\[ y''(x)=y(x) \left (-\csc ^2(x)\right ) \left (-\left (\left (a^2 b^2-(a+1)^2\right ) \sin ^2(x)\right )-a (a+1) b \sin (2 x)+(1-a) a\right ) \] ✓ Mathematica : cpu = 0.59845 (sec), leaf count = 166
DSolve[Derivative[2][y][x] == -(Csc[x]^2*((1 - a)*a - (-(1 + a)^2 + a^2*b^2)*Sin[x]^2 - a*(1 + a)*b*Sin[2*x])*y[x]),y[x],x]
\[\left \{\left \{y(x)\to c_2 \left (e^{-a b x} \sin ^{-a-1}(x)+\frac {2^{2 a+1} (2 a+1) \left (1-e^{2 i x}\right )^{2 a} \left (-i e^{-i x} \left (-1+e^{2 i x}\right )\right )^{-2 a} e^{-a b x+2 i x} \sin ^a(x) \operatorname {Hypergeometric2F1}\left (2 a+2,i b a+a+1,i b a+a+2,e^{2 i x}\right ) (b \sin (x)+\cos (x))}{a (b-i)-i}\right )+c_1 e^{a b x} \sin ^a(x) (b \sin (x)+\cos (x))\right \}\right \}\] ✓ Maple : cpu = 0.839 (sec), leaf count = 98
dsolve(diff(diff(y(x),x),x) = -(-(a^2*b^2-(a+1)^2)*sin(x)^2-a*(a+1)*b*sin(2*x)-a*(a-1))/sin(x)^2*y(x),y(x))
\[y \left (x \right ) = \left (c_{2} \left (\cot \left (x \right )+i\right )^{\frac {1}{2}+\frac {1}{2} a +\frac {1}{2} i a b} \operatorname {hypergeom}\left (\left [i a b -a +1, a \left (i b +1\right )\right ], \left [i a b +a +2\right ], \frac {1}{2}-\frac {i \cot \left (x \right )}{2}\right )+c_{1} \left (\cot \left (x \right )+i\right )^{-\frac {1}{2}-\frac {1}{2} i a b -\frac {1}{2} a} \left (b +\cot \left (x \right )\right )\right ) \left (\cot \left (x \right )-i\right )^{-\frac {1}{2}+\frac {1}{2} i a b -\frac {1}{2} a}\]