\[ y'(x) \cos (a y(x))-b (1-c \cos (a y(x))) \sqrt {c \cos (a y(x))+\cos ^2(a y(x))-1}=0 \] ✓ Mathematica : cpu = 4.12004 (sec), leaf count = 369
DSolve[-(b*(1 - c*Cos[a*y[x]])*Sqrt[-1 + c*Cos[a*y[x]] + Cos[a*y[x]]^2]) + Cos[a*y[x]]*Derivative[1][y][x] == 0,y[x],x]
\[\left \{\left \{y(x)\to \text {InverseFunction}\left [-\frac {i (\cos (\text {$\#$1} a)+1) \sqrt {\frac {2 c \cos (\text {$\#$1} a)+\cos (2 \text {$\#$1} a)-1}{(\cos (\text {$\#$1} a)+1)^2}} \sqrt {\frac {c \tan ^2\left (\frac {\text {$\#$1} a}{2}\right )+\sqrt {c^2+4}+2}{\sqrt {c^2+4}+2}} \sqrt {1-\frac {c \tan ^2\left (\frac {\text {$\#$1} a}{2}\right )}{\sqrt {c^2+4}-2}} \left ((c-1) F\left (i \sinh ^{-1}\left (\sqrt {-\frac {c}{\sqrt {c^2+4}-2}} \tan \left (\frac {a \text {$\#$1}}{2}\right )\right )|\frac {2-\sqrt {c^2+4}}{\sqrt {c^2+4}+2}\right )+2 \Pi \left (\frac {(c+1) \left (\sqrt {c^2+4}-2\right )}{(c-1) c};i \sinh ^{-1}\left (\sqrt {-\frac {c}{\sqrt {c^2+4}-2}} \tan \left (\frac {a \text {$\#$1}}{2}\right )\right )|\frac {2-\sqrt {c^2+4}}{\sqrt {c^2+4}+2}\right )\right )}{a \left (c^2-1\right ) \sqrt {\frac {c}{4-2 \sqrt {c^2+4}}} \sqrt {2 c \cos (\text {$\#$1} a)+\cos (2 \text {$\#$1} a)-1} \sqrt {-c \tan ^4\left (\frac {\text {$\#$1} a}{2}\right )-4 \tan ^2\left (\frac {\text {$\#$1} a}{2}\right )+c}}\& \right ]\left [-\frac {b x}{\sqrt {2}}+c_1\right ]\right \}\right \}\] ✓ Maple : cpu = 0.432 (sec), leaf count = 48
dsolve(diff(y(x),x)*cos(a*y(x))-b*(1-c*cos(a*y(x)))*(cos(a*y(x))^2-1+c*cos(a*y(x)))^(1/2) = 0,y(x))
\[x +c_{1}+\int _{}^{y \left (x \right )}\frac {2 \cos \left (\textit {\_a} a \right )}{b \left (c \cos \left (\textit {\_a} a \right )-1\right ) \sqrt {-2+2 \cos \left (2 \textit {\_a} a \right )+4 c \cos \left (\textit {\_a} a \right )}}d \textit {\_a} = 0\]