\[ \int \frac {\left (-1+x^4\right ) \sqrt [4]{x^2+x^6}}{1+x^4+x^8} \, dx \]
Optimal antiderivative \[ \frac {\arctan \! \left (\frac {x}{\left (x^{6}+x^{2}\right )^{\frac {1}{4}}}\right )}{2}-\frac {\arctan \! \left (\frac {\sqrt {2}\, x \left (x^{6}+x^{2}\right )^{\frac {1}{4}}}{-x^{2}+\sqrt {x^{6}+x^{2}}}\right ) \sqrt {2}}{4}-\frac {\operatorname {arctanh}\! \left (\frac {x}{\left (x^{6}+x^{2}\right )^{\frac {1}{4}}}\right )}{2}+\frac {\operatorname {arctanh}\! \left (\frac {\frac {\sqrt {2}\, x^{2}}{2}+\frac {\sqrt {x^{6}+x^{2}}\, \sqrt {2}}{2}}{x \left (x^{6}+x^{2}\right )^{\frac {1}{4}}}\right ) \sqrt {2}}{4} \]
command
Int[((-1 + x^4)*(x^2 + x^6)^(1/4))/(1 + x^4 + x^8),x]
Rubi 4.17.3 under Mathematica 13.3.1 output
\[ \text {\$Aborted} \]
Rubi 4.16.1 under Mathematica 13.3.1 output
\[ \frac {2 \left (-\sqrt {3}+i\right ) x \sqrt [4]{x^6+x^2} \operatorname {AppellF1}\left (\frac {3}{8},-\frac {1}{4},1,\frac {11}{8},-x^4,-\frac {2 x^4}{1-i \sqrt {3}}\right )}{3 \left (\sqrt {3}+i\right ) \sqrt [4]{x^4+1}}+\frac {2 \left (\sqrt {3}+i\right ) x \sqrt [4]{x^6+x^2} \operatorname {AppellF1}\left (\frac {3}{8},-\frac {1}{4},1,\frac {11}{8},-x^4,-\frac {2 x^4}{1+i \sqrt {3}}\right )}{3 \left (-\sqrt {3}+i\right ) \sqrt [4]{x^4+1}} \]