\[ y''(x)=\left (a^2-n^2\right ) y(x)-2 n \coth (x) y'(x) \] ✓ Mathematica : cpu = 0.659368 (sec), leaf count = 273
DSolve[Derivative[2][y][x] == (a^2 - n^2)*y[x] - 2*n*Coth[x]*Derivative[1][y][x],y[x],x]
\[\left \{\left \{y(x)\to \frac {c_2 (-1)^{\frac {1}{2} (-2 n-1)+1} \tanh ^2(x)^{\frac {1}{4} (-2 n-1)+1} \left (\tanh ^2(x)-1\right )^{\frac {1}{2} \left (\frac {a+n}{2}+\frac {1}{2} (a+n+1)+\frac {1}{2} (-2 n-1)+1\right )} \, _2F_1\left (\frac {1}{2} (-2 n-1)+\frac {a+n}{2}+1,\frac {1}{2} (-2 n-1)+\frac {1}{2} (a+n+1)+1;\frac {1}{2} (-2 n-1)+2;\tanh ^2(x)\right ) \exp \left (\frac {1}{2} (n-1) \log \left (1-\tanh ^2(x)\right )-n \log (\tanh (x))\right )}{\sqrt {\tanh (x)}}+\frac {c_1 \tanh ^2(x)^{\frac {1}{4} (2 n+1)} \left (\tanh ^2(x)-1\right )^{\frac {1}{2} \left (\frac {a+n}{2}+\frac {1}{2} (a+n+1)+\frac {1}{2} (-2 n-1)+1\right )} \, _2F_1\left (\frac {a+n}{2},\frac {1}{2} (a+n+1);\frac {1}{2} (2 n+1);\tanh ^2(x)\right ) \exp \left (\frac {1}{2} (n-1) \log \left (1-\tanh ^2(x)\right )-n \log (\tanh (x))\right )}{\sqrt {\tanh (x)}}\right \}\right \}\] ✓ Maple : cpu = 0.135 (sec), leaf count = 36
dsolve(diff(diff(y(x),x),x) = -2*n/sinh(x)*cosh(x)*diff(y(x),x)-(-a^2+n^2)*y(x),y(x))
\[y \left (x \right ) = \left (\sinh ^{-n +\frac {1}{2}}\left (x \right )\right ) \left (\LegendreP \left (a -\frac {1}{2}, n -\frac {1}{2}, \cosh \left (x \right )\right ) c_{1}+\LegendreQ \left (a -\frac {1}{2}, n -\frac {1}{2}, \cosh \left (x \right )\right ) c_{2}\right )\]