\[ \int \sec ^{\frac {3}{2}}(c+d x) (a+a \sec (c+d x))^{5/2} \left (A+B \sec (c+d x)+C \sec ^2(c+d x)\right ) \, dx \]
Optimal antiderivative \[ \frac {a^{\frac {5}{2}} \left (400 A +326 B +283 C \right ) \arcsinh \left (\frac {\sqrt {a}\, \tan \left (d x +c \right )}{\sqrt {a +a \sec \left (d x +c \right )}}\right )}{128 d}+\frac {a \left (2 B +C \right ) \left (\sec ^{\frac {5}{2}}\left (d x +c \right )\right ) \left (a +a \sec \left (d x +c \right )\right )^{\frac {3}{2}} \sin \left (d x +c \right )}{8 d}+\frac {C \left (\sec ^{\frac {5}{2}}\left (d x +c \right )\right ) \left (a +a \sec \left (d x +c \right )\right )^{\frac {5}{2}} \sin \left (d x +c \right )}{5 d}+\frac {a^{3} \left (400 A +326 B +283 C \right ) \left (\sec ^{\frac {3}{2}}\left (d x +c \right )\right ) \sin \left (d x +c \right )}{128 d \sqrt {a +a \sec \left (d x +c \right )}}+\frac {a^{3} \left (1040 A +950 B +787 C \right ) \left (\sec ^{\frac {5}{2}}\left (d x +c \right )\right ) \sin \left (d x +c \right )}{960 d \sqrt {a +a \sec \left (d x +c \right )}}+\frac {a^{2} \left (80 A +110 B +79 C \right ) \left (\sec ^{\frac {5}{2}}\left (d x +c \right )\right ) \sin \left (d x +c \right ) \sqrt {a +a \sec \left (d x +c \right )}}{240 d} \]
command
integrate(sec(d*x+c)^(3/2)*(a+a*sec(d*x+c))^(5/2)*(A+B*sec(d*x+c)+C*sec(d*x+c)^2),x, algorithm="maxima")
Maxima 5.46 SBCL 2.0.1.debian via sagemath 9.6 output
\[ \text {output too large to display} \]
Maxima 5.44 via sagemath 9.3 output \[ \text {Timed out} \]_____________________________________________________