\[ \int \cos ^{\frac {11}{2}}(c+d x) (a+b \sec (c+d x))^3 \left (A+B \sec (c+d x)+C \sec ^2(c+d x)\right ) \, dx \]
Optimal antiderivative \[ \frac {2 \left (7 a^{3} B +27 B a \,b^{2}+3 b^{3} \left (3 A +5 C \right )+3 a^{2} b \left (7 A +9 C \right )\right ) \sqrt {\frac {\cos \left (d x +c \right )}{2}+\frac {1}{2}}\, \EllipticE \left (\sin \left (\frac {d x}{2}+\frac {c}{2}\right ), \sqrt {2}\right )}{15 \cos \left (\frac {d x}{2}+\frac {c}{2}\right ) d}+\frac {2 \left (165 a^{2} b B +77 b^{3} B +33 a \,b^{2} \left (5 A +7 C \right )+5 a^{3} \left (9 A +11 C \right )\right ) \sqrt {\frac {\cos \left (d x +c \right )}{2}+\frac {1}{2}}\, \EllipticF \left (\sin \left (\frac {d x}{2}+\frac {c}{2}\right ), \sqrt {2}\right )}{231 \cos \left (\frac {d x}{2}+\frac {c}{2}\right ) d}+\frac {2 \left (24 A \,b^{3}+77 a^{3} B +242 B a \,b^{2}+33 a^{2} b \left (7 A +9 C \right )\right ) \left (\cos ^{\frac {3}{2}}\left (d x +c \right )\right ) \sin \left (d x +c \right )}{495 d}+\frac {2 a \left (24 A \,b^{2}+143 a b B +9 a^{2} \left (9 A +11 C \right )\right ) \left (\cos ^{\frac {5}{2}}\left (d x +c \right )\right ) \sin \left (d x +c \right )}{693 d}+\frac {2 \left (6 A b +11 B a \right ) \left (\cos ^{\frac {3}{2}}\left (d x +c \right )\right ) \left (b +a \cos \left (d x +c \right )\right )^{2} \sin \left (d x +c \right )}{99 d}+\frac {2 A \left (\cos ^{\frac {3}{2}}\left (d x +c \right )\right ) \left (b +a \cos \left (d x +c \right )\right )^{3} \sin \left (d x +c \right )}{11 d}+\frac {2 \left (165 a^{2} b B +77 b^{3} B +33 a \,b^{2} \left (5 A +7 C \right )+5 a^{3} \left (9 A +11 C \right )\right ) \sin \left (d x +c \right ) \left (\sqrt {\cos }\left (d x +c \right )\right )}{231 d} \]
command
integrate(cos(d*x+c)^(11/2)*(a+b*sec(d*x+c))^3*(A+B*sec(d*x+c)+C*sec(d*x+c)^2),x, algorithm="fricas")
Fricas 1.3.8 (sbcl 2.2.11.debian) via sagemath 9.6 output
\[ \frac {2 \, {\left (315 \, A a^{3} \cos \left (d x + c\right )^{4} + 75 \, {\left (9 \, A + 11 \, C\right )} a^{3} + 2475 \, B a^{2} b + 495 \, {\left (5 \, A + 7 \, C\right )} a b^{2} + 1155 \, B b^{3} + 385 \, {\left (B a^{3} + 3 \, A a^{2} b\right )} \cos \left (d x + c\right )^{3} + 45 \, {\left ({\left (9 \, A + 11 \, C\right )} a^{3} + 33 \, B a^{2} b + 33 \, A a b^{2}\right )} \cos \left (d x + c\right )^{2} + 77 \, {\left (7 \, B a^{3} + 3 \, {\left (7 \, A + 9 \, C\right )} a^{2} b + 27 \, B a b^{2} + 9 \, A b^{3}\right )} \cos \left (d x + c\right )\right )} \sqrt {\cos \left (d x + c\right )} \sin \left (d x + c\right ) - 15 \, \sqrt {2} {\left (5 i \, {\left (9 \, A + 11 \, C\right )} a^{3} + 165 i \, B a^{2} b + 33 i \, {\left (5 \, A + 7 \, C\right )} a b^{2} + 77 i \, B b^{3}\right )} {\rm weierstrassPInverse}\left (-4, 0, \cos \left (d x + c\right ) + i \, \sin \left (d x + c\right )\right ) - 15 \, \sqrt {2} {\left (-5 i \, {\left (9 \, A + 11 \, C\right )} a^{3} - 165 i \, B a^{2} b - 33 i \, {\left (5 \, A + 7 \, C\right )} a b^{2} - 77 i \, B b^{3}\right )} {\rm weierstrassPInverse}\left (-4, 0, \cos \left (d x + c\right ) - i \, \sin \left (d x + c\right )\right ) - 231 \, \sqrt {2} {\left (-7 i \, B a^{3} - 3 i \, {\left (7 \, A + 9 \, C\right )} a^{2} b - 27 i \, B a b^{2} - 3 i \, {\left (3 \, A + 5 \, C\right )} b^{3}\right )} {\rm weierstrassZeta}\left (-4, 0, {\rm weierstrassPInverse}\left (-4, 0, \cos \left (d x + c\right ) + i \, \sin \left (d x + c\right )\right )\right ) - 231 \, \sqrt {2} {\left (7 i \, B a^{3} + 3 i \, {\left (7 \, A + 9 \, C\right )} a^{2} b + 27 i \, B a b^{2} + 3 i \, {\left (3 \, A + 5 \, C\right )} b^{3}\right )} {\rm weierstrassZeta}\left (-4, 0, {\rm weierstrassPInverse}\left (-4, 0, \cos \left (d x + c\right ) - i \, \sin \left (d x + c\right )\right )\right )}{3465 \, d} \]
Fricas 1.3.7 via sagemath 9.3 output
\[ {\rm integral}\left ({\left (C b^{3} \cos \left (d x + c\right )^{5} \sec \left (d x + c\right )^{5} + {\left (3 \, C a b^{2} + B b^{3}\right )} \cos \left (d x + c\right )^{5} \sec \left (d x + c\right )^{4} + A a^{3} \cos \left (d x + c\right )^{5} + {\left (3 \, C a^{2} b + 3 \, B a b^{2} + A b^{3}\right )} \cos \left (d x + c\right )^{5} \sec \left (d x + c\right )^{3} + {\left (C a^{3} + 3 \, B a^{2} b + 3 \, A a b^{2}\right )} \cos \left (d x + c\right )^{5} \sec \left (d x + c\right )^{2} + {\left (B a^{3} + 3 \, A a^{2} b\right )} \cos \left (d x + c\right )^{5} \sec \left (d x + c\right )\right )} \sqrt {\cos \left (d x + c\right )}, x\right ) \]