[{\cite[Theorem~5.14]{Guterman2025}}]\label{MainTheoremEvenIntro}Assume that $k \ge 4, n \ge k + 2, n + k$ is even and $|{\mathbb F}| > k$. Let $T\colon \M_{n\,k} ({\mathbb F}) \to \M_{n\,k} ({\mathbb F})$ be a linear map. Then $\det_{n\,k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\,k} ({\mathbb F})$ if and only if there exist $A \in \M_{n\,n}({\mathbb F})$ and $B \in \M_{k\,k}({\mathbb F})$ such that
\[
\det_{n\,k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\]
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and
\[
T(X) = AXB
\]
for all $X \in \M_{n\,k} ({\mathbb F}).$

The theory of linear maps preserving matrix invariants is well-known direction of research in Linear Algebra connected with other fields of mathematics. It has a long history which dates back to the beginning of 20th century in the works of Frobenius~\cite{GF} and continues at present time (see~\cite{LAMA199233} for the survey of the results until the end of 20th century and~\cite{hogben_handbook_2014} for a brief introduction).

The Cullis' determinant is an invariant of rectangular matrix generalizing the notion of the ordinary determinant of square matrix which was introduced by Cullis in~\cite{cullis1913} and studied later by Radi\'{c} in~\cite{radic1966,radic2005,radic1991,radic2008}, Makarewicz, Mozgawa, Pikuta and Sza\l{}kowski in~\cite{makarewicz2014,makarewicz2016,makarewicz2020}, Amiri, Fathy and Bayat in~\cite{amiri2010}, and Nakagami and Yanay in~\cite{NAKAGAMI2007422}. Thus, the question regarding the description of linear maps preserving the Cullis' determinant arises naturally. The detailed introduction to the subject could be found in~\cite{Guterman2025}.
In particular, \cite{Guterman2025} contains the description of linear maps preserving the Cullis determinant $\det_{n\,k}$ for the case when $ k \ge 4$, $n \ge k + 2$ and $n + k$ is even as it stated in the following theorem (all necessary definitions are given in Section~\ref{sec:prelim}).

The Cullis' determinant is an invariant of rectangular matrix generalizing the notion of the ordinary determinant of square matrix which was introduced by Cullis in~\cite{cullis1913} and studied later by Radi\'{c} in~\cite{radic1966,radic2005,radic1991,radic2008}, Makarewicz, Mozgawa, Pikuta and Sza\l{}kowski in~\cite{makarewicz2014,makarewicz2016,makarewicz2020}, Amiri, Fathy and Bayat in~\cite{amiri2010}, and Nakagami and Yanay in~\cite{NAKAGAMI2007422}. Thus, the question regarding the description of linear maps preserving the Cullis' determinant arises naturally. The detailed introduction to the subject could be found in~\cite{Guterman2025}.
In particular, \cite{Guterman2025} contains the description of linear maps preserving the Cullis determinant $\det_{n\,k}$ for the case when $ k \ge 4$, $n \ge k + 2$ and $n + k$ is even as it stated in the following theorem (all necessary definitions are given in Section~\ref{sec:prelim}).

The parity of $n + k$ is significant because if $n + k$ is odd, then there exist linear maps preserving the Cullis' determinant which do not have such form (Corollary~\ref{cor:AdditionLinearMapNPLusKOdd}).

\begin{theorem}[Theorem~\ref{thm:MainTheoremCullisNKOdd}]Assume that $|\F| > k \ge 4$, $n \ge k + 2$ and $n + k$ is odd. Let $T\colon \M_{n\,k} (\F) \to \M_{n\,k} (\F)$ be a linear map. Then $\det_{n\, k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\,k} (\F)$ if and only if there exist $A \in \M_{n\, n}(\F)$ and $B \in \M_{k\, k}(\F)$ such that
\[
\det_{n\,k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\]
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and a linear map\linebreak $\phi\colon \M_{n\,k}(\F) \to W_{n\,k}$ such that
\[
\det_{n\,k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\]
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and
\begin{equation*}
T(X) = AXB + \phi(X)
\end{equation*}
for all $X \in \M_{n\,k} (\F).$ Here $W_{n\,k} \subseteq \M_{n\,k}(\F)$ denotes the space of matrices, all rows of which are equal, being a radical of $\det_{n\,k}$.
\end{theorem}

\begin{lemma}[{\cite[Lemma~5.13]{Guterman2025}}]\label{lem:CullisDeg1Rank1}Assume that $|\F| > k \ge 4$, $n \ge k + 2$ and $n + k$ is even. Let $T\colon \M_{n\, k} (\F) \to \M_{n\, k} (\F)$ be a linear map such that $\det_{n\, k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\, k} (\F).$ Then $\rk (X) = 1$ implies $\rk (T(X)) = 1.$
\end{lemma}

We omit the subscripts for $\sgn_{[n]}$ and $\sgn_{n\,k}$ if this cannot lead to a misunderstanding.\begin{definition}[\cite{NAKAGAMI2007422}, Theorem 13]\label{def:CullisDet} Let $n \ge k$, $X \in \M_{n\,k} (\mathbb F)$. Then Cullis' determinant $\det_{n\, k}(X)$ of $X$ is defined to be the function:
$$
\det_{n\, k} (X) = \sum_{\sigma \in \mathcal C_{k}^{[n]}} \sgn_{n\,k} (\sigma) x_{\sigma(1)\, 1}x_{\sigma(2)\, 2}\ldots x_{\sigma(k)\, k}.
$$
We also denote $\det_{n\, k} (X)$ as follows
\[
\det_{n\,k}(X)=\begin{vmatrix}x_{1\,1} & \cdots & x_{1\,k}\\
\vdots & \cdots & \vdots\\
x_{n\,1} & \cdots & x_{n\,k}
\end{vmatrix}_{n\,k}.
\]
If $n = k$, then we also write $\det_{k}$ or $\det$ instead of $\det_{n\,k}$ because in this case $\det_{n\,k}$ is clearly equal to an ordinary determinant of a square matrix.
\end{definition}

\begin{lemma}[{\cite[Lemma~3.1]{Guterman2025}}]\label{lem:TwoSidedMulPreservesDet}Let $n \ge k$, $A \in \M_{n\,n}(\F)$, $B \in \M_{k\,k}(\F)$, and let $T\colon \M_{n\,k}(\F) \to \M_{n\,k}(\F)$ be a linear map defined by
\begin{equation}\label{eq:TwoSideMul}
T(X) = AXB
\end{equation}
for all $X \in \M_{n\,k}(\F).$ Then
\[
\det_{n\,k} (T(X)) = \det_{n\,k}(X)
\]
for all $X \in \M_{n\,k} (\F)$ if and only if
\begin{equation}\label{eq:TwoSideMul2}
\det_{n\,k} \Bigl(A(|d]\Bigr)\cdot \det_k \Bigl(B\Bigr) = \sgn (d)
\end{equation}
for all $d \in \binom{[n]}{k}.$
\end{lemma}

\begin{lemma}[{\cite[Theorem~5.14]{Guterman2025}}]\label{lem:MainTheoremEven}Assume that $|\F| > k \ge 4, n \ge k + 2$ and $n + k$ is even. Let $T\colon \M_{n\, k} (\F) \to \M_{n\, k} (\F)$ be a linear map. Then $\det_{n\, k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\, k} (\F)$ if and only if there exist $A \in \M_{n\, n}(\F)$ and $B \in \M_{k\, k}(\F)$ such that
\begin{equation}\label{thm:MainTheoremEvenKGe4:eq}
\det_{n\, k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \cdot \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\end{equation}
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and
\[
T(X) = AXB
\]
for all $X \in \M_{n\, k} (\F).$
\end{lemma}

\begin{theorem}\label{thm:MainTheoremCullisNKOdd}Assume that $|\F| > k \ge 4$, $n \ge k + 2$ and $n + k$ is odd. Let $T\colon \M_{n\,k} (\F) \to \M_{n\,k} (\F)$ be a linear map. Then $\det_{n\, k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\,k} (\F)$ if and only if there exist $A \in \M_{n\, n}(\F)$ and $B \in \M_{k\, k}(\F)$ such that
\begin{equation}\label{thm:MainTheoremCullisNKOdd:eqq}
\det_{n\,k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\end{equation}
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and a linear map\linebreak $\phi\colon \M_{n\,k}(\F) \to W_{n\,k}$ such that
\begin{equation}\label{thm:MainTheoremCullisNKOdd:eq}
T(X) = AXB + \phi(X)
\end{equation}
for all $X \in \M_{n\,k} (\F).$

\begin{corollary}\label{cor:AdditionLinearMapNPLusKOdd}Suppose that $n + k$ is odd. Let
$\phi \colon \M_{n\,k}(\F) \to W_{n\,k}$
be a linear map. Denote by $S$ a linear map $\M_{n}(\F) \to \M_{n\,k}(\F)$ defined by
\begin{equation}\label{eq:AddingKernel}
S(X) = X + \phi (X)
\end{equation}
for all $X \in \M_{n\,k}(\F).$ Then
\[
\det_{n\, k} (S(X)) = \det_{n\,k}(X)
\]
for all $X \in \M_{n\,k} (\F).$
\end{corollary}

\begin{theorem}\label{thm:MainTheoremCullisNKOdd}Assume that $|\F| > k \ge 4$, $n \ge k + 2$ and $n + k$ is odd. Let $T\colon \M_{n\,k} (\F) \to \M_{n\,k} (\F)$ be a linear map. Then $\det_{n\, k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\,k} (\F)$ if and only if  there exist $A \in \M_{n\, n}(\F)$ and $B \in \M_{k\, k}(\F)$ such that
\begin{equation}\label{thm:MainTheoremCullisNKOdd:eqq}
\det_{n\,k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\end{equation}
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and a linear map\linebreak $\phi\colon \M_{n\,k}(\F) \to W_{n\,k}$ such that
\begin{equation}\label{thm:MainTheoremCullisNKOdd:eq}
T(X) = AXB + \phi(X)
\end{equation}
for all $X \in \M_{n\,k} (\F).$

Here $W_{n\,k} \subseteq \M_{n\,k}(\F)$ is the space of matrices, all rows of which are equal, which is defined in Definition~\ref{def:WNK} and  is a radical of $\det_{n\,k}$ by Lemma~\ref{lem:NKOddRadDetNK}.
\end{theorem}

The parity of $n + k$ is significant because if $n + k$ is odd, then there exist linear maps preserving the Cullis' determinant which do not have such form (Corollary~\ref{cor:AdditionLinearMapNPLusKOdd}).

However, in this paper we show that the case $(n,k)$ with $n \ge k$ and $n + k$ odd reduces to the case $(n-1,k)$ (Lemma~\ref{lem:FromNKOddToNKEvenIfKerW}\ref{lem:FromNKOddToNKEvenIfKerW:part6}). Using this reduction we prove the main theorem of this paper providing the description of linear maps preserving $\det_{n\,k}$ if $n + k$ is odd.

\begin{theorem}[Theorem~\ref{thm:MainTheoremCullisNKOdd}]Assume that $|{\mathbb F}| > k \ge 4$, $n \ge k + 2$ and $n + k$ is odd. Let $T\colon \M_{n\,k} ({\mathbb F}) \to \M_{n\,k} ({\mathbb F})$ be a linear map. Then $\det_{n\, k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\,k} ({\mathbb F})$ if and only if  there exist $A \in \M_{n\, n}({\mathbb F})$ and $B \in \M_{k\, k}({\mathbb F})$ such that
\[
\det_{n\,k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\]
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and a linear map\linebreak $\phi\colon \M_{n\,k}({\mathbb F}) \to W_{n\,k}$ such that
\[
\det_{n\,k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\]
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and
\begin{equation*}
T(X) = AXB + \phi(X)
\end{equation*}
for all $X \in \M_{n\,k} ({\mathbb F}).$ Here $W_{n\,k} \subseteq \M_{n\,k}({\mathbb F})$ denotes the space of matrices, all rows of which are equal, being a radical of $\det_{n\,k}$.
\end{theorem}

Our proof of the main theorem relies on the notion of the radical of a function which was introduced and studied by Waterhouse~\cite{Waterhouse1983} (Definition~\ref{def:radical} below). The reduction of the case $(n,k)$ with $k \ge 4, n \ge k + 2$ and $n + k$ odd to the case $(n-1,k)$ with $k \ge 4, n \ge k + 2$ and $(n-1) + k$ even is done by finding the radical of $\det_{n\,k}$ explicitly and using its properties.

Thus, the results of the current paper and its predecessor~\cite{Guterman2025} provide the solution to linear preserver problem for the Cullis' determinant if $k \ge 4, n \ge k + 2$ and the ground field is large enough. The cases $k = 1$ and $k = 2$ are considered in~\cite{Guterman2025} as well. The case $n = k + 1$ is considered in~\cite{Guterman2024}, where the first attempt to find the description of linear maps preserving the Cullis' determinant is made. The remaining case $k = 3$ need a special approach because the proof for the case $k \ge 4$ relies on the following lemma and will be discussed separately. We remark that in this paper we apply the results obtained in~\cite{Guterman2025} and we do not use any results from~\cite{Guterman2024}.

\begin{lemma}[{\cite[Lemma~5.13]{Guterman2025}}]\label{lem:CullisDeg1Rank1}Assume that $|{\mathbb F}| > k \ge 4$, $n \ge k + 2$ and $n + k$ is even. Let  $T\colon \M_{n\, k} ({\mathbb F}) \to \M_{n\, k} ({\mathbb F})$ be a linear map such that  $\det_{n\, k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\, k} ({\mathbb F}).$ Then $\rk (X) = 1$ implies $\rk (T(X)) = 1.$
\end{lemma}

The example below shows that this lemma does not hold for $k = 3$.

\begin{example}[{\cite[Example~5.11]{Guterman2025}}]\label{cex:K3NKEvenInt}Let $B = \begin{psmallmatrix}
                                 1 & 0 & 0 & 0 & -1 & 0 & \cdots & 0\\
                                 0 & 1 & 0 & -1 & 0 & 0 & \cdots & 0\\
                                 0 & 0 & 0 & 0 & 0 & 0 & \cdots & 0
                                        \end{psmallmatrix}^t \in \M_{n\,3}({\mathbb F}).$ Then $\rk (B) = 2$ and $\deg_{\lambda} (\det_{n\,k}(A + \lambda B)) \le 1$ for all $A \in \M_{n\,3}({\mathbb F})$.
\end{example}

Nevertheless, if $k = 3$, then linear maps preserving the Cullis determinant admit the same description as in the case when $k \ge 4$ and will be studied separately.\bigskip

The paper is organized as follows: in Section~\ref{sec:prelim} we provide the basic facts regarding $\det_{n\,k}$; in Section~\ref{sec:kernels} we find the radical of $\det_{n\,k}$ if $n + k$ is odd; in Section~\ref{sec:maintheorem} we reduce the linear preserver problem for $\det_{n\,k}$ and $n + k$ is odd to the linear preserver problem for $\det_{(n-1)\,k}$ and prove the main characterization theorem for $n + k$ odd.

The introduction explains that “the parity of $n+k$ is significant” since when $n+k$ is odd “there exist linear maps preserving the Cullis' determinant which do not have such form.” It then outlines a reduction argument: “the case $(n,k)$ with $n\ge k$ and $n+k$ odd reduces to the case $(n-1,k)$” (via Lemma~\ref{lem:FromNKOddToNKEvenIfKerW}…), and “using this reduction we prove the main theorem … for $n+k$ … odd.”

It further specifies what drives this reduction: “Our proof of the main theorem relies on the notion of the radical of a function … (Definition~\ref{def:radical} below). The reduction … is done by finding the radical of $\det_{n\,k}$ explicitly and using its properties.” In particular, for $n+k$ odd one identifies the radical space $W_{n\,k}$ (“the space of matrices, all rows of which are equal, being a radical of $\det_{n\,k}$”), which leads to the possibility of an additive radical term $\phi(X)$ in the odd-parity characterization.

The introduction explains that “the parity of $n+k$ is significant” since when $n+k$ is odd “there exist linear maps preserving the Cullis' determinant which do not have such form.” It then outlines a reduction argument: “the case $(n,k)$ with $n\ge k$ and $n+k$ odd reduces to the case $(n-1,k)$” (via the following lemma).

\begin{lemma}\label{lem:FromNKOddToNKEvenIfKerW}Assume that  $|\F| > k$, $n > k$ and $n + k$ is odd. Let $T\colon \M_{n\, k} (\F) \to \M_{n\, k} (\F)$ be a linear map such that $\det_{n\, k} (T(X)) = \det_{n\,k}(X)$ and $\Img(T) = \M^0_{n\,k}(\F)$. Then the following statements hold.
\begin{enumerate}[label=(\alph*), ref=(\alph*)]
\item\label{lem:FromNKOddToNKEvenIfKerW:part4} $\Ker(T) = \rad(\det_{n\,k})$.
\item\label{lem:FromNKOddToNKEvenIfKerW:part5} $L^{+}\circ L^{-} \circ T = T$.
\item\label{lem:FromNKOddToNKEvenIfKerW:part6} $T \circ L^{+} \circ L^{-} = T$.
\item\label{lem:FromNKOddToNKEvenIfKerW:part2} If a linear map $S$ is defined by
\begin{equation}\label{lem:FromNKOddToNKEvenIfKerW:eq}
S = L^{-} \circ T \circ L^{+},
\end{equation}
then $T = L^{+} \circ S \circ L^{-}$.
\end{enumerate}
\end{lemma}

Using this reduction we prove the main theorem for $n+k$ odd.

It further specifies what drives this reduction: “Our proof of the main theorem relies on the notion of the radical of a function …” where the radical is defined as follows.

\begin{definition}[Cf.~{\cite[text at the beginning of Section~1]{Waterhouse1983}}]\label{def:radical}Let $\F$ be a field, $V$ be a finite-dimensional vector space over $\F$, and let $f$ be a function from $V$ to $\F$. The \emph{radical} of $f$, denoted by $\rad (f)$ is a subset of $V$ defined by
\[
\rad (f) = \{\mathbf w \mid f(\mathbf v + \lambda \mathbf w) = f(\mathbf v)\;\;\mbox{for all}\;\; \mathbf v \in V,\;\; \lambda \in \F\}.
\]
\end{definition}

The reduction is done by finding the radical of $\det_{n\,k}$ explicitly and using its properties. In particular, for $n+k$ odd one identifies the radical space $W_{n\,k}$ (“the space of matrices, all rows of which are equal, being a radical of $\det_{n\,k}$”), which leads to the possibility of an additive radical term $\phi(X)$ in the odd-parity characterization.

[{Guterman2025, Theorem~5.14} ]\label{MainTheoremEvenIntro}Assume that $k \ge 4, n \ge k + 2, n + k$ is even and $|{\mathbb F}| > k$. Let $T\colon \M_{n\,k} ({\mathbb F}) \to \M_{n\,k} ({\mathbb F})$ be a linear map. Then $\det_{n\,k} (T(X)) = \det_{n\,k}(X)$ for all $X \in \M_{n\,k} ({\mathbb F})$ if and only if there exist $A \in \M_{n\,n}({\mathbb F})$ and $B \in \M_{k\,k}({\mathbb F})$ such that
\[
\det_{n\,k} \Bigl(A(|i_1,\ldots, i_k]\Bigr) \det_k \Bigl(B\Bigr) = (-1)^{i_1 + \ldots + i_k - 1 - \ldots - k}
\]
for all increasing sequences $1 \le i_1 < \ldots < i_k \le n $ and
\[
T(X) = AXB
\]
for all $X \in \M_{n\,k} ({\mathbb F}).$

Biconditional or Equivalence
Existential–Universal

Question: Assume that $k\ge 4$, $n\ge k+2$, $n+k$ is even, and $|\mathbb F|>k$. Let $T\colon \mathrm M_{n,k}(\mathbb F)\to \mathrm M_{n,k}(\mathbb F)$ be a linear map, where $\det_{n,k}$ denotes the Cullis determinant on $n\times k$ matrices. For $A\in \mathrm M_{n,n}(\mathbb F)$ and indices $1\le i_1<\cdots<i_k\le n$, write $A(|i_1,\ldots,i_k])$ for the $n\times k$ submatrix of $A$ formed by columns $i_1,\ldots,i_k$. Which of the following statements is equivalent to the condition that $\det_{n,k}(T(X))=\det_{n,k}(X)$ for all $X\in \mathrm M_{n,k}(\mathbb F)$?

There exist $A\in \mathrm M_{n,n}(\mathbb F)$ and $B\in \mathrm M_{k,k}(\mathbb F)$ such that
\[
\det_{n,k}\bigl(A(|i_1,\ldots,i_k])\bigr)\,\det_k(B)=(-1)^{i_1+\cdots+i_k-1-\cdots-k}
\]
for all increasing sequences $1\le i_1<\cdots<i_k\le n$, and
\[
T(X)=AXB
\]
for all $X\in \mathrm M_{n,k}(\mathbb F)$.

Choices:
B.
There exist $A\in \mathrm M_{n,n}(\mathbb F)$ and $B\in \mathrm M_{k,k}(\mathbb F)$, and a linear map $\phi\colon \mathrm M_{n,k}(\mathbb F)\to W_{n,k}$, such that
\[
\det_{n,k}\bigl(A(|i_1,\ldots,i_k])\bigr)\,\det_k(B)=(-1)^{i_1+\cdots+i_k-1-\cdots-k}
\]
for all increasing sequences $1\le i_1<\cdots<i_k\le n$, and
\[
T(X)=AXB+\phi(X)
\]
for all $X\in \mathrm M_{n,k}(\mathbb F)$, where $W_{n,k}\subseteq \mathrm M_{n,k}(\mathbb F)$ is the space of matrices all of whose rows are equal.
C.
There exist $A\in \mathrm M_{n,n}(\mathbb F)$ and $B\in \mathrm M_{k,k}(\mathbb F)$ such that
\[
\det_{n,k}\bigl(A(|i_1,\ldots,i_k])\bigr)\,\det_k(B)=(-1)^{i_1+\cdots+i_k-1-\cdots-k}
\]
for all increasing sequences $1\le i_1<\cdots<i_k\le n$, and
\[
T(X)=AXB
\]
for all rank-one matrices $X\in \mathrm M_{n,k}(\mathbb F)$.
D.
There exist $A\in \mathrm M_{n,n}(\mathbb F)$ and $B\in \mathrm M_{k,k}(\mathbb F)$ such that
\[
\det_{n,k}\bigl(A(|i_1,\ldots,i_k])\bigr)\,\det_k(B)=(-1)^{i_1+\cdots+i_k-1-\cdots-k}
\]
for some increasing sequence $1\le i_1<\cdots<i_k\le n$, and
\[
T(X)=AXB
\]
for all $X\in \mathrm M_{n,k}(\mathbb F)$.
E.
There exist $A\in \mathrm M_{n,n}(\mathbb F)$ and $B\in \mathrm M_{k,k}(\mathbb F)$ such that
\[
\det_{n,k}\bigl(A(|i_1,\ldots,i_k])\bigr)\,\det_k(B)=(-1)^{i_1+\cdots+i_k-1-\cdots-k}
\]
for all increasing sequences $1\le i_1<\cdots<i_k\le n$, and either
\[
T(X)=AXB
\]
or
\[
T(X)=AX^{\top}B
\]
for all $X\in \mathrm M_{n,k}(\mathbb F)$.

universality over all matrices dropped to rank-one inputs

minor sign condition required for all increasing sequences

two-sided multiplication form replaced by transpose alternative


For almost all irrational $\theta\in [0,1]$, if we denote by $q_k/p_k$ the sequence of convergents of the continued fraction expansion of $\theta$, then one has
$$\lim_{k\to \infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}=\frac{16}{\pi^3}\sum_{n\ge 1 \textrm{ odd}}\frac{1}{n^3\sin(n\pi\theta)}.$$

\begin{Remark}
  {\em
The $\mathrm{SU}_2$-TQFT $\boV_p$ corresponds in Conformal Field Theory to the Wess-Zumino-Witten model at odd level $\ell=p-2$. Similar $\mathrm{SU}_2$-TQFTs exist at  even levels, that is, when $p$ is even. Their signature functions can again be computed using the Frobenius algebra formalism of Theorem~\ref{DM1}.  But we won't consider even level $\mathrm{SU}_2$-TQFTs in this paper.
}\end{Remark}

\section{A trigonometric formula for the signature in genus 2}\label{genre2}
Let us now start our investigation of the asymptotic behavior of the signature function. The first result of this paper is the following trigonometric formula for the signature in genus $2$.

\begin{Theorem}\label{sigma2}
  For $0<q<p$ coprime odd integers, we have
\begin{equation}\nonumber \sigma_2({{\textstyle\frac q p}})= \frac {1-p^2}{6p^2} +
  \frac 1 {4p^2}\sum_{n=1,\, \mathrm{ odd}}^{p-2}\frac{f(n,p,q)}{\sin^3(\frac{n\pi}{2p})\sin^2(\frac{qn\pi}{p})}
\end{equation}
where
\begin{eqnarray*} f(n,p,q)&=(3p-3)\sin(\frac{(2q-1)n\pi}{2p})+(p+1)\sin(\frac{(2q-3)n\pi}{2p})\\
&+(p-1)\sin(\frac{(2q+3)n\pi}{2p})+(3p+3)\sin(\frac{(2q+1)n\pi}{2p}).
\end{eqnarray*}
\end{Theorem}

\begin{proof}
By a finite Fourier transform over $\Z/2p\Z$, we can write for $m\in\Z$ :
\begin{equation}\label{FFT}
  (-1)^{\lfloor m/p\rfloor}=
  \frac 1 p
  \sum_{n=1,\text{ odd}}^{2p-1}\frac{\sin(\pi n(2m+1)/2p)}{\sin(\pi n/2p)}
  =\frac 1 p  \sum_{n=1,\text{ odd}}^{2p-1} \frac {t^{2m+1}-t^{-2m-1}}{t-t^{-1}}
    \Big|_{t=e^{\frac {i\pi n}{2p}}~.
      }
\end{equation}
In the sequel, we set $\O_p=\{1,3,\ldots,2p-1\}$ and $\O_p^+=\{1,3,\ldots,p\}$. Using Formula \eqref{FFT} for each sign $\epsilon_{j+1}=(-1)^{\lfloor q(j+1)/p\rfloor}$ and plugging it into the formula of Lemma \ref{triplepsilon}, we get
\begin{equation}\nonumber\sigma_2({\textstyle{\frac q p}})\,\,
  =\sum_{(j,k,\ell)\in T_p} \epsilon_{j+1}\epsilon_{k+1}\epsilon_{\ell+1}
  =\frac1 {p^3} \sum_{(a,b,c)\in \O_p^3}\psi(a,b,c)\end{equation}
where
\begin{equation}
\psi(a,b,c)=\!\!\!\!\sum\limits_{(j,k,l)\in T_p}\!\!\!\!
\frac{(x^{2q(j+1)+1}-x^{-2q(j+1)-1})(y^{2q(k+1)+1}-y^{-2q(k+1)-1})(z^{2q(\ell+1)+1}-z^{-2q(\ell+1)-1})}{(x-x^{-1})(y-y^{-1})(z-z^{-1})}\nonumber
\end{equation}
with $x=e^{\frac{i\pi a}{2p}}$,  $y=e^{\frac{i\pi b}{2p}}$ and $z=e^{\frac{i\pi c}{2p}}$.

The function $\psi$ is clearly symmetric in its three variables and satisfies also $\psi(2p-a,b,c)=\psi(a,b,c)$. We can hence rewrite $\sigma_2(\frac q p)$  as
\begin{equation}\label{eq4n}\sigma_2({\textstyle{\frac q p}})=
  \frac1 {p^3}
  \sum_{a,b,c\in \O_p^+} 2^{m_a+m_b+m_c}\psi(a,b,c)\end{equation}
where $m_a=1$ if $a\ne p$ and $m_p=0$.

In order to compute $\psi(a,b,c)$, we rewrite it as follows:
\begin{equation} \psi(a,b,c)= \sum\limits_{\epsilon,\mu,\nu=\pm 1}\epsilon\mu\nu \frac{ x^{\epsilon}y^{\mu}z^{\nu}F(x^{2q\epsilon},y^{2q\mu},z^{2q\nu})}{(x-x^{-1})(y-y^{-1})(z-z^{-1})}\label{eq5n}
\end{equation} where $F$ is the polynomial defined by $F(X,Y,Z)=\sum_{(j,k,l)\in T_p}X^{j+1}Y^{k+1}Z^{\ell +1}$.

The computation of $F(x^{2q\epsilon},y^{2q\mu},z^{2q\nu})$ is simplified by the following observation:  we have $$F(X,Y,Z)=X^pY^p F(X^{-1}, Y^{-1},Z)~,$$ since the involution $(j,k,\ell ) \mapsto (p-2-j,p-2-k,\ell)$ preserves the set $T_p$. Therefore, since $F$ is symmetric in its three variables,  whenever $F(X,Y,Z)$ is evaluated at numbers $X=x^{2q}$, $Y=y^{2q}$, and $Z=z^{2q}$
satisfying $X^p=Y^p=Z^p=-1$, then for  any $\epsilon,\mu,\nu=\pm 1$, we have that $F(X^\epsilon, Y^\mu,Z^\nu)$ is equal to either $F(X,Y,Z)$ (when $\epsilon\mu\nu =1$) or to  $F(X^{-1}, Y^{-1},Z^{-1})= \overline {F(X,Y,Z)}$ (when $\epsilon\mu\nu =-1$).
\begin{Remark}\label{rk33}{\em For later use, we note the following special case: when $F(x^{2q},y^{2q},z^{2q})$ happens to be real, then $F(x^{2q\epsilon},y^{2q\mu},z^{2q\nu})$
    does not depend at all on $\epsilon,\mu$, or $\nu$, and formula \eqref{eq5n} for $\psi(a,b,c)$ simply becomes $\psi(a,b,c)=F(x^{2q},y^{2q},z^{2q})$.}
    \end{Remark}

Let $p$ be an odd positive integer and $\mathbb{Q}(\zeta)$ be the cyclotomic field generated by a primitive root of unity $\zeta$ of order $2p$. The SU$_2$-TQFT constructs for any surface $S_g$ of genus $g$ a finite dimensional $\mathbb{Q}(\zeta)$-vector space $\boV_p(S_g)$ endowed with an Hermitian form $h_g$ defined over $\mathbb{Q}(\zeta)$. These vector spaces are part of a modular functor, meaning that they support a projective unitary action of the corresponding mapping class groups and satisfy a list of compatibilities when cutting the surfaces along simple closed curves.

The dimension of $\boV_p(S_g)$, given by the celebrated Verlinde formula, has received considerable attention since the nineties. On the other hand, taking $0<q<p$ two coprime odd integers, one can embed $\mathbb{Q}(\zeta)$ into $\mathbb{C}$ by sending $\zeta$ to $e^{i\pi q/p}$. The Hermitian form $h_g$ has a signature that we denote by
$$\sigma_g(\textstyle{\frac{q}{p}})=\Sign\Big(\boV_p(S_g)\!\!\underset{\zeta=e^{i\pi q/p}}{\otimes}\!\!\mathbb{C},\,h_g\Big)$$
and which has been very little studied. It was shown in \cite{DM} that for a fixed $\zeta=e^{i\pi q/p}$, these signatures have cut and paste properties very similar to those of the dimensions, which allows to encode them in a single Frobenius algebra $V_{\frac q p}$. Later it was proved in \cite{S2B} that $V_{\frac q p}$ is closely related to the hyperbolic geometry of the two-bridge knot $K(p,q)$. The purpose of this paper is to study signatures in a different direction, by analyzing the asymptotic behavior of $\sigma_{g}(\frac{q}{p})$ when $\frac{q}{p}\to \theta$ for a fixed $\theta\in [0,1]$.

If we take $q=1$, the Hermitian form $h_g$ becomes positive definite,
so that $\sigma_g(\frac 1 p)$ coincides with the dimension of $\boV_p(S_g)$. The problem of computing the asymptotic behavior of the dimensions when $p\to \infty$ is well-known, and related to considerations of semi-classical analysis on character varieties $\mathcal{M}_g=\Hom(\pi_1(S_g),\SU_2)/\SU_2$. This led Witten \cite{Witten} to show that for $g>1$,
$$\lim_{p\to \infty}\frac{\dim \boV_p(S_g)}{p^{3g-3}}=\operatorname{Vol}(\mathcal{M}_g)=2(2\pi^2)^{1-g}\zeta(2g-2).$$

In genus $0$ and $1$, the hermitian form is positive definite for all values of $q$, so let us consider the first non trivial case $g=2$. In this case, one can write a simple formula for $\sigma_2(\frac q p)$:
\begin{equation}\label{defsigma2}
 \sigma_2({\textstyle{\frac{q}{p}}})=\sum_{(j,k,\ell)\in \Delta_p}
 (-1)^{\lfloor {jq}/{p}\rfloor +\lfloor {kq}/{p}\rfloor+\lfloor {\ell q}/{p}\rfloor}
\end{equation}
 where the sum runs over the set $\Delta_p$ of triples of integers satisfying
$$\begin{cases}0<j,k,\ell<p, \\
j<k+\ell,k<j+\ell,\ell<j+k,\\
j+k+\ell<2p,\textrm{ and } j+k+\ell\textrm{ odd}.
\end{cases}$$

Our main result is the following

Our main result is the following

Figure \ref{fig:sigma2} suggests that the convergence is rapid: the graph of the map on the right hand side is shown in green whereas red dots represent the points $(q/p,\sigma_2(q/p)/p^2)$ for $0<q<p<31$.

We briefly review the properties of the convergents $q_k/p_k$ of $\theta$ that we will need. See {\em e.g.} Perron \cite{P} or Khinchin \cite{Kh1} for more details\footnote{Unfortunately, \cite{Kh1} writes $p_k$ for our $q_k$ and {\em vice versa.}}. Each irrational $\theta\in {\mathbb R}$ has a unique regular continued fraction expansion $$\theta = [a_0; a_1, a_2, \ldots ] = a_0+\cfrac{1}{a_1+\cfrac{1}{a_2+\dots}} $$ where regular means that $a_0\in \Z$ and $a_i\in \Z_{\geq 1}$ for $i\geq 1$. Note that in our case $a_0=0$ since
$\theta \in [0,1]\setminus \Q$.
The convergents
$$\frac{q_k}{p_k} = [a_0; a_1, a_2, \ldots, a_k] = a_0+\cfrac{1}{a_1+\cfrac{1}{\dots+ \frac{1}{a_k}}} $$ can also be defined recursively by
\begin{eqnarray} q_k&=a_k q_{k-1} + q_{k-2}, \ \ \ \ q_0=a_0, \ \ \ \ q_{-1}=1 \label{rec5}\\
 p_k&=a_k p_{k-1} + p_{k-2}, \ \ \ \ p_0=1, \ \ \ \ p_{-1}=0 \label{rec6}
\end{eqnarray}
The following classical facts are well-known.
\begin{Proposition}\label{qpfacts}
 For all irrational $\theta$, the convergents $q_k/p_k$ of $\theta$ satisfy the following properties:
 \begin{enumerate}
 \item[(i)] One has $1=p_0\leq p_1<p_2<\ldots$, {\em i.e.} the denominators $p_k$ are positive integers and moreover strictly increasing from $p_1$ onwards.
 \item[(ii)] For all $k\geq 1$, $q_k$ and $p_k$ are coprime.
 \item[(iii)] $\lim_{k\rightarrow\infty} q_k/p_k = \theta$.
 \item[(iv)]
 For all $i\geq 1$, we have
 \begin{equation}\nonumber
 \frac 1 {p_i(p_i+p_{i+1})} <
 \Big|\theta - \frac {q_i}{p_i} \Big| < \frac 1 {p_ip_{i+1}} < \frac 1 {p_i^2}~.
 \end{equation}
 \item[(v)] {\em [Lagrange]}\footnote{For a proof, see Perron \cite[\S 15, Satz 2.17]{P}.} For all $1\leq i<k$, and any $n,\ell \in\Z$ with $1\leq n<p_{i+1}$, we have
 \begin{equation}\nonumber
 \Big| n\frac {q_k}{p_k} -\ell \Big|
 \geq \Big|p_i\frac {q_k}{p_k} - q_i\Big| \geq \frac{1}{p_i + p_{i+1}} ~.
 \end{equation}
 \end{enumerate}
 \end{Proposition}

\begin{proof}[\bf{Proof of Theorem~\ref{asympto}}] We start from the trigonometric formula for $\sigma_2(\frac{q}{p})$ given in Theorem~\ref{sigma2}. The first observation is that the term $(1-p^2)/(6p^2)$ in this formula stays bounded as $p\rightarrow \infty$ and so does not contribute to the limit of $\sigma_2(\frac{q_k}{p_k})/ p_k^2$. Thus it suffices to show
 \begin{equation}\label{eq4}
 \lim_{k\rightarrow \infty} \frac{1}{4p_k^4} \sum_{n=1,\textrm{ odd}}^{p_k-2}\frac{f(n,p_k,q_k)}{\sin^3({n\pi}/{2p_k})\sin^2({q_kn\pi}/{p_k})} =\frac{16}{\pi^3}\sum_{n\ge 1, \textrm{odd}}\frac{1}{n^3\sin(n\pi\theta)}
 \end{equation} with $f(n,p,q)$ given as in Theorem~\ref{sigma2}.

Let $S^{(1)}(q_k,p_k,\theta)$ denote the sum $S^{(1)}(q_k,p_k)$ but with $\sin(n\pi\theta)$ in the denominator in place of $\sin({q_kn\pi}/{p_k})$. Using (\ref{sin-ineq}), we see that the $n$-th summand of $S^{(1)}(q_k,p_k,\theta)$ is bounded from above by $1/(n^3\sin(n\pi\theta))$. So we can apply the Lebesgue dominated convergence theorem to this sum. We get
 $$\lim_{k\rightarrow \infty}S^{(1)}(q_k,p_k,\theta) =\frac{8}{\pi^3}\sum_{n\ge 1, \textrm{odd}}\frac{1}{n^3\sin(n\pi\theta)}~.$$
 Thus to prove Lemma~\ref{3.3} it suffices to show that the difference $d_k=S^{(1)}(q_k,p_k) -S^{(1)}(q_k,p_k,\theta)$ goes to zero as $k\rightarrow \infty$. We have
 $$d_k= \frac{1}{p_k^3} \sum_{n=1,\textrm{ odd}}^{p_k-2}\frac{\cos (n\pi/2p_k)}{\sin^3({n\pi}/{2p_k}) }\Biggl (\frac {1} {\sin({q_kn\pi}/{p_k})} - \frac {1} {\sin(n\pi\theta)}\Biggr)~.$$ We can bound the denominators from below using (\ref{sin-ineq}), (\ref{sinn-inequ}), and the trivial lower bound $|\sin({q_kn\pi}/{p_k})| \geq \sin (\pi/p_k) \geq 2/p_k$. Thus for all $\epsilon >0$ we have

\begin{proof}[\bf{Proof of Lemma~\ref{3.4}}]
 We assume that $\mu(\theta)=2$ and that $\theta$ satisfies the conditions
 in Theorem~\ref{ThKL}.
 Recall $S^{(2)}(q_k,p_k)$, defined in (\ref{sum2}), is a sum from $n=1$ to $n=p_k-2$. We cut
it into two parts, as follows. Put
\begin{equation}\label{floor} N(k)= \lfloor 3k/4 \rfloor
\end{equation} and write $$S^{(2)}(q_k,p_k) = S^{(2.1)}(q_k,p_k)
+S^{(2.2)}(q_k,p_k)~,$$ where $S^{(2.1)}(q_k,p_k)$ is the sum from
$n=1$ to $n=p_{N(k)+1}-1$ and $S^{(2.2)}(q_k,p_k)$ is the sum from
$n=p_{N(k)+1}$ to $p_k-2$. The following two lemmas provide upper bounds for these two sums.
\begin{Lemma}\label{3.9} We have $$|S^{(2.1)}(q_k,p_k) |\leq \frac
{p_{N(k)}} {p_k} F_7(k)$$ where $F_7(k) ={\mathcal O}(k^7)$.
\end{Lemma}
\begin{Lemma}\label{3.10} We have $$|S^{(2.2)}(q_k,p_k)| \leq \frac{1}{2} \zeta(2)
\frac {p_{k}} {p_{N(k)+1}^2}~.$$
\end{Lemma}
Assuming these two lemmas for the moment, since $\lim_{m\rightarrow\infty} p_m^{1/m}= L>1$ by the Khinchin-L{\'e}vy Theorem~\ref{ThKL}(ii), $p_k$ grows like $L^k$ as $k\rightarrow \infty$. It is not hard to deduce from this that both $ S^{(2.1)}(q_k,p_k)$ and $ S^{(2.2)}(q_k,p_k)$ converge to zero.~\footnote{Here is how to see this for $ S^{(2.1)}(q_k,p_k)$. Theorem~\ref{ThKL}(ii) gives that for all $\epsilon>0$ there exist $m_0(\epsilon)$ so that $L^{1-\epsilon} \leq p_m^{1/m} \leq L^{1+\epsilon}$ holds for for all $m\geq m_0(\epsilon)$. Thus also $L^{m-m\epsilon} \leq p_m\leq L^{m+m\epsilon}$ for all $m\geq m_0(\epsilon)$. Applying now the first inequality with $m=k$ and the second one with $m=N(k)=\lfloor 3k/4\rfloor$ and taking $\epsilon$ small enough, it follows that $p_{N(k)}/p_k \leq L^{-\delta k}$ for some $\delta>0$ as soon as $k\geq 2 m_0(\epsilon)$. Thus $p_{N(k)} F_7(k)/ {p_k} \leq k^7 L^{-\delta k}$ for $k$ big enough, proving that the limit as $k\rightarrow\infty$ is zero, as asserted. }
This proves Lemma~\ref{3.4}.
\end{proof}
It remains to prove Lemmas~\ref{3.9} and~\ref{3.10}. Let us first deal
with Lemma~\ref{3.10} whose proof is easy.
\begin{proof}[\bf{Proof of Lemma~\ref{3.10}}] Since $n\geq p_{N(k)+1}$
 in this sum,
 we can bound the
$\sin^2(n\pi/2p_k)$ term in the denominator from below by
$p_{N(k)+1}^2/p_k^2$. Thus
\begin{equation}\label{sum3}|S^{(2.2)}(q_k,p_k)| \leq \frac 1 {p_k
p_{N(k)+1}^2}\sum_{n=1,\textrm{ odd}}^{p_k-2} \frac 1
{\sin^2(q_kn\pi/p_k)}~.\end{equation}

\section{The higher genus case}\label{sec5}
In this section, we briefly discuss how the asymptotic behaviour of the genus $2$ signature might generalize to higher genus. Numerical experiments (see below) indicate that
 $\sigma_{g,n}(\frac{q}{p};\lambda)$ might grow like $p^{\max\{g,2g-2\}}$ when $\frac q p$ goes to an irrational $\theta\in[0,1]$. So we ask the following question.
\begin{Question}\label{51}
 Given $g,n\in \Z_{\geq 0}$ and $n$-tuples of integers $\lambda=(\lambda_1,\ldots,\lambda_n)\in \Z^n_{\geq 0}$,
 is there a function $F_{g;\lambda}:
 [0,1]
 \to \R$ defined almost everywhere satisfying the following:
 For almost all irrational $\theta\in[0,1]$,
 if we write $\frac{q_k}{p_k}$ for the convergents of $\theta$, one has
$$ \lim_{k\to\infty}\frac{\sigma_{g,n}(\frac{q_k}{p_k};\lambda)}{p_k^{\max\{g,2g-2\}}}=F_{g;\lambda}(\theta)$$
\end{Question}
Theorem \ref{asympto} shows that the answer is yes for $g=2$ and $n=0$. Let us
compare this to the asymptotics of the dimension
of the TQFT vector spaces
which
is well-known: it has a semi-classical interpretation for which we refer to \cite[Section 3]{Witten}. Precisely:
$$ \lim_{p\to\infty}\frac{\dim \boV_p(S_g)}{p^{\max\{g,3g-3\}}}=\operatorname{Vol}(\mathcal{M}_g)=2(2\pi^2)^{1-g}\zeta(2g-2)$$
where the last equality holds only for $g>0$ and $\mathcal{M}_g$ is the character variety of representations of $\pi_1(S_g)$ into SU$_2$, endowed with its Liouville measure. We have no conceptual explanation why the order of growth $3g-3$
for the dimension
should be replaced by $2g-2$
for the signature.

Figure \ref{fig:sigma2}  suggests that the convergence is rapid: the graph of the map on the right hand side is shown in green whereas red dots represent the points $(q/p,\sigma_2(q/p)/p^2)$ for $0<q<p<31$.

\begin{figure}[htbp]
\begin{center}
\includegraphics[width=10cm]{genre2.png}
\caption{Graph of the signature function $\sigma_2$}\label{fig:sigma2}
\end{center}
\end{figure}

The proof of this theorem involves several steps and occupies Sections~\ref{genre2} and~\ref{sec2}. We first apply in Section~\ref{genre2} a finite Fourier transform in Equation \eqref{defsigma2} to transform this expression into a sum of evaluations of a rational function at roots of unity indexed by $\Delta_p$. We then apply Brion's theorem to get an explicit trigonometric sum computing $\sigma_2(q/p)$ (see Theorem \ref{sigma2} for the precise statement): this step already presents miraculous cancellations for which we do not have any conceptual explanation.

Then in Section~\ref{sec2}, we analyse the asymptotic behavior of the trigonometric sum in the most technical part of the paper. In order to control its denominators, we need to impose that the coefficients of the continued fraction expansion of $\theta$ do not grow too quickly. We use for that  some classical results of Khinchin on metric Diophantine approximation and a variation on a technique due to Zaremba.

In higher genus one can also write down a formula for
  $\sigma_g(q/p)$ as the sum of signs indexed by lattice points inside some polytope (see \cite[Remark 4.12]{BHMV}~\footnote{Our $\boV_p$ corresponds to the $V_{2p}$ of \cite{BHMV}.}.) But the signs are much more complicated than in Formula \eqref{defsigma2} which prevents us from generalizing our theorem. We briefly discuss in Section~\ref{sec5} how the asymptotics might look like for general surfaces including marked points, and illustrate this by some numerical experiments.

In the last section of the paper, we discuss some modular properties of the signature. We observe that the function
$$\Lambda(\theta)=\frac{16}{\pi^3}\sum_{n\ge 1\textrm{ odd}}\frac{1}{n^3\sin(n\pi\theta)}$$
is the boundary value of an Eichler integral of a modular form of weight $4$ for the level $2$ group $\Gamma(2)$. In addition to the obvious property $\Lambda(\theta+1)=-\Lambda(\theta)$, it satisfies the following transformation law:
\begin{equation}\label{Lambdamod}
\Lambda\Big(\frac{\theta}{2\theta+1}\Big)(2\theta+1)^2-\Lambda(\theta)=2\theta^2+2\theta+1.
\end{equation}
This functional equation suggests that a similar integral version holds for signatures as stated in the following conjecture, checked by computer for all $0<q<p<100$.
\begin{Conjecturestar}
For any odd coprime integers $0<q<p$ one has
$$\sigma_2\Big(\frac{q}{2q+p}\Big)-\sigma_2\Big(\frac{q}{p}\Big)=2q^2+2qp+p^2-1.$$
\end{Conjecturestar}

This formula is reminiscent of the reciprocity formula of Dedekind sums: in fact we will make this analogy more precise in Section~\ref{SectionS}. We prove that the signature at $\zeta=e^{i\pi q/p}$ of  the TQFT-vector space associated to the 4 times punctured sphere (with  colors $(p-1)/2$  at the marked points)  is actually equal to a 2-smoothed version, $S(q/p)$,  of the Dedekind sum $s(q,p)$. It is then classical that  this function $S(q/p)$ is the boundary value of a modular form (this time of weight $1/2$) and that it satisfies the following transformation law

$$S\Big(\frac{q}{2q+p}\Big)-S\Big(\frac{q}{p}\Big)=1.$$
We provide an elementary proof of these facts for the benefit of the non-specialist reader.

This very satisfactory state of things would hopefully generalize to higher genus as well. However, even in genus 2, a precise (non asymptotic) relation between the signature $\sigma_g(q/p)$
and the Eichler integral is still lacking. Such a relation would presumably lead to a proof  of the conjectured transformation law for $\sigma_2$.

To end this introduction, we would like to mention the parallel paper \cite{MY} in which the first author and S. Yoon compute the asymptotics of the function
$\sigma_g(q/p)$, but for sequences of the form $\frac{q_n}{p_n}=\frac{a+bn}{c+dn}$ where $\left(\begin{smallmatrix} a & b \\ c & d \end{smallmatrix}\right)\in \Gamma(2)$. This regime is somewhat orthogonal to the one described in the present paper. The techniques used and the results are also very different. A synthesis of the two papers has yet to be made.
\vskip 8pt

{\bf Acknowledgements:} It is our pleasure to thank Pierre Charollois for early discussions on many aspects of this paper, Gaëtan Chenevier for his expertise on modular forms,
and Don Zagier for very helpful explanations about Eichler integrals and quantum modular forms.

The proof “involves several steps and occupies Sections~\ref{genre2} and~\ref{sec2}.” First, “we apply in Section~\ref{genre2} a finite Fourier transform in Equation \eqref{defsigma2} to transform this expression into a sum of evaluations of a rational function at roots of unity indexed by $\Delta_p$.” Next, “we then apply Brion's theorem to get an explicit trigonometric sum computing $\sigma_2(q/p)$ (see Theorem \ref{sigma2} for the precise statement),” a step that “already presents miraculous cancellations.” Then, “in Section~\ref{sec2}, we analyse the asymptotic behavior of the trigonometric sum in the most technical part of the paper.” To “control its denominators,” one “need[s] to impose that the coefficients of the continued fraction expansion of $\theta$ do not grow too quickly,” using “classical results of Khinchin on metric Diophantine approximation and a variation on a technique due to Zaremba.”

The proof “involves several steps and occupies” the part starting at

\begin{Remark}
  {\em
The $\mathrm{SU}_2$-TQFT $\boV_p$ corresponds in Conformal Field Theory to the Wess-Zumino-Witten model at odd level $\ell=p-2$. Similar $\mathrm{SU}_2$-TQFTs exist at  even levels, that is, when $p$ is even. Their signature functions can again be computed using the Frobenius algebra formalism of Theorem~\ref{DM1}.  But we won't consider even level $\mathrm{SU}_2$-TQFTs in this paper.
}\end{Remark}

\section{A trigonometric formula for the signature in genus 2}\label{genre2}
Let us now start our investigation of the asymptotic behavior of the signature function. The first result of this paper is the following trigonometric formula for the signature in genus $2$.

\begin{Theorem}\label{sigma2}
  For $0<q<p$ coprime odd integers, we have
\begin{equation}\nonumber \sigma_2({{\textstyle\frac q p}})= \frac {1-p^2}{6p^2} +
  \frac 1 {4p^2}\sum_{n=1,\, \mathrm{ odd}}^{p-2}\frac{f(n,p,q)}{\sin^3(\frac{n\pi}{2p})\sin^2(\frac{qn\pi}{p})}
\end{equation}
where
\begin{eqnarray*} f(n,p,q)&=(3p-3)\sin(\frac{(2q-1)n\pi}{2p})+(p+1)\sin(\frac{(2q-3)n\pi}{2p})\\
&+(p-1)\sin(\frac{(2q+3)n\pi}{2p})+(3p+3)\sin(\frac{(2q+1)n\pi}{2p}).
\end{eqnarray*}
\end{Theorem}

and the later section \ref{sec2}.

First, “we apply” a finite Fourier transform in the defining expression
\begin{equation}\label{defsigma2}
  \sigma_2({\textstyle{\frac{q}{p}}})=\sum_{(j,k,\ell)\in \Delta_p}
 (-1)^{\lfloor {jq}/{p}\rfloor +\lfloor {kq}/{p}\rfloor+\lfloor {\ell q}/{p}\rfloor}
\end{equation}
to transform this expression into a sum of evaluations of a rational function at roots of unity indexed by $\Delta_p$.

Next, “we then apply Brion's theorem to get an explicit trigonometric sum computing $\sigma_2(q/p)$,” namely the formula stated in
\begin{Theorem}\label{sigma2}
  For $0<q<p$ coprime odd integers, we have
\begin{equation}\nonumber \sigma_2({{\textstyle\frac q p}})= \frac {1-p^2}{6p^2} +
  \frac 1 {4p^2}\sum_{n=1,\, \mathrm{ odd}}^{p-2}\frac{f(n,p,q)}{\sin^3(\frac{n\pi}{2p})\sin^2(\frac{qn\pi}{p})}
\end{equation}
where
\begin{eqnarray*} f(n,p,q)&=(3p-3)\sin(\frac{(2q-1)n\pi}{2p})+(p+1)\sin(\frac{(2q-3)n\pi}{2p})\\
&+(p-1)\sin(\frac{(2q+3)n\pi}{2p})+(3p+3)\sin(\frac{(2q+1)n\pi}{2p}).
\end{eqnarray*}
\end{Theorem}

a step that “already presents miraculous cancellations.” Then, “we analyse the asymptotic behavior of the trigonometric sum in the most technical part of the paper.” To “control its denominators,” one “need[s] to impose that the coefficients of the continued fraction expansion of $\theta$ do not grow too quickly,” using “classical results of Khinchin on metric Diophantine approximation and a variation on a technique due to Zaremba.”

Question:
For a reduced rational number \(q/p\in(0,1)\), define
\[
\sigma_2\!\left(\frac qp\right)=\sum_{(a,b,c)\in \Delta_p}(-1)^{\lfloor aq/p\rfloor+\lfloor bq/p\rfloor+\lfloor cq/p\rfloor},
\]
where \(\Delta_p\) is the set of triples \((a,b,c)\) of integers such that
\[
0<a,b,c<p,\qquad a<b+c,\quad b<a+c,\quad c<a+b,
\]
\[
a+b+c<2p,\qquad a+b+c\text{ is odd}.
\]
Now let \(\theta\in[0,1]\) be irrational, and let \(q_k/p_k\) be the convergents of the regular continued-fraction expansion of \(\theta\). Which statement holds for Lebesgue-almost every irrational \(\theta\in[0,1]\)?

The limit exists and is given by
\[
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
=\frac{16}{\pi^3}\sum_{n\ge 1\,\mathrm{odd}}\frac{1}{n^3\sin(n\pi\theta)}.
\]

Choices:
B.
The limit exists and is given by
\[
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
=\frac{16}{\pi^3}\sum_{n\ge 1}\frac{1}{n^3\sin(n\pi\theta)}.
\]
C.
For Lebesgue-almost every irrational \(\theta\in[0,1]\), the limit
\[
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
\]
exists.
D.
The limit exists and is given by
\[
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
=\frac{16}{\pi^3}\sum_{n\ge 1\,\mathrm{odd}}\frac{1}{n^3\,|\sin(n\pi\theta)|}.
\]
E.
For every irrational \(\theta\in[0,1]\), if \(q_k/p_k\) are the convergents of its regular continued-fraction expansion, then
\[
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
=\frac{16}{\pi^3}\sum_{n\ge 1\,\mathrm{odd}}\frac{1}{n^3\sin(n\pi\theta)}.
\]

odd-index restriction in the trigonometric expansion

explicit identification of the limiting value by the odd-series formula

signed denominator \(\sin(n\pi\theta)\) replaced by absolute value, destroying the exact Fourier-cancellation structure

almost-everywhere metric Diophantine hypothesis strengthened to all irrationals


Question:
For integers $p>0$ and $0<q<p$, define
\[
\sigma_2\!\left(\frac{q}{p}\right)
=\sum_{(a,b,c)\in \Delta_p}
(-1)^{\lfloor aq/p\rfloor+\lfloor bq/p\rfloor+\lfloor cq/p\rfloor},
\]
where $\Delta_p$ is the set of triples $(a,b,c)\in \mathbb Z^3$ such that
\[
0<a,b,c<p,\qquad a<b+c,\ b<a+c,\ c<a+b,
\]
\[
a+b+c<2p,\qquad a+b+c\text{ is odd}.
\]
Let $\theta\in[0,1]$ be irrational, and let $q_k/p_k$ be the convergents of the regular continued fraction expansion of $\theta$. For almost all irrational $\theta\in[0,1]$, as $k\to\infty$, which limiting statement holds?

\[
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
=\frac{16}{\pi^3}\sum_{n\ge 1\ \mathrm{odd}}\frac{1}{n^3\sin(n\pi\theta)}.
\]

Choices:
B.
\[
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
=\frac{16}{\pi^3}\sum_{n\ge 1}\frac{1}{n^3\sin(n\pi\theta)}.
\]
C.
\[
\text{For almost all irrational }\theta\in[0,1],\quad
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
\text{ exists.}
\]
D.
\[
\text{For every irrational }\theta\in[0,1],\qquad
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^2}
=\frac{16}{\pi^3}\sum_{n\ge 1\ \mathrm{odd}}\frac{1}{n^3\sin(n\pi\theta)}.
\]
E.
\[
\lim_{k\to\infty}\frac{\sigma_2(q_k/p_k)}{p_k^3}
=\frac{16}{\pi^3}\sum_{n\ge 1\ \mathrm{odd}}\frac{1}{n^3\sin(n\pi\theta)}.
\]

explicit identification of the limit by the trigonometric series