\[ \boxed { \left \{ {\frac {{\rm d}^{2}}{{\rm d}{t}^{2}}}x \left ( t \right ) -a{\frac {\rm d}{{\rm d}t}}y \left ( t \right ) +bx \left ( t \right ) =0,{\frac {{\rm d}^{2}}{{\rm d}{t}^{2}}}y \left ( t \right ) +a{\frac {\rm d}{{\rm d}t}}x \left ( t \right ) +by \left ( t \right ) =0 \right \} } \]
Mathematica: cpu = 0.370547 (sec), leaf count = 4815 \[ \left \{\left \{x(t)\to \frac {e^{-\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}-\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \left (e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2-e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2-e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2+e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t} a^2+\sqrt {a^2 \left (a^2+4 b\right )} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}\right ) c_1}{4 \sqrt {a^2 \left (a^2+4 b\right )}}+\frac {e^{-\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}-\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \left (\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2-\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2+\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2-\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t} a^2-\sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}-\sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}\right ) c_2}{2 \sqrt {2} \sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}}}-\frac {a b e^{-\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}-\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \left (-e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}}+e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}}+\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}-\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}\right ) c_3}{\sqrt {2} \sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}}}-\frac {a e^{-\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}-\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \left (-e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}-e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}\right ) c_4}{2 \sqrt {a^2 \left (a^2+4 b\right )}},y(t)\to \frac {a b e^{-\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}-\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \left (-e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}}+e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}}+\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}-\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}\right ) c_1}{\sqrt {2} \sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}}}+\frac {a e^{-\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}-\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \left (-e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}-e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}\right ) c_2}{2 \sqrt {a^2 \left (a^2+4 b\right )}}+\frac {e^{-\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}-\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \left (e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2-e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2-e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2+e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t} a^2+\sqrt {a^2 \left (a^2+4 b\right )} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}\right ) c_3}{4 \sqrt {a^2 \left (a^2+4 b\right )}}+\frac {e^{-\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}-\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} \left (\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2-\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2+\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}} a^2-\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t} a^2-\sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}-\sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} e^{\sqrt {2} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t+\frac {\sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}}+\sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} e^{\frac {\sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} t}{\sqrt {2}}+\sqrt {2} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}} t}\right ) c_4}{2 \sqrt {2} \sqrt {a^2 \left (a^2+4 b\right )} \sqrt {-a^2-2 b-\sqrt {a^2 \left (a^2+4 b\right )}} \sqrt {-a^2-2 b+\sqrt {a^2 \left (a^2+4 b\right )}}}\right \}\right \} \]
Maple: cpu = 0.094 (sec), leaf count = 868 \[ \left \{ \left \{ x \left ( t \right ) ={\it \_C1}\,{{\rm e}^{-{\frac {t }{2}\sqrt {-2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) } -4\,b}}}}+{\it \_C2}\,{{\rm e}^{{\frac {t}{2}\sqrt {-2\,{a}^{2}-2\, \sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}}}}+{\it \_C3}\,{ {\rm e}^{-{\frac {t}{2}\sqrt {-2\,{a}^{2}+2\,\sqrt {{a}^{2} \left ( {a} ^{2}+4\,b \right ) }-4\,b}}}}+{\it \_C4}\,{{\rm e}^{{\frac {t}{2}\sqrt {-2\,{a}^{2}+2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}}}}, y \left ( t \right ) =-{\frac {1}{8\,ab} \left ( {\it \_C4}\, \left ( -2\, {a}^{2}+2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b \right ) ^ {{\frac {3}{2}}}{{\rm e}^{{\frac {t}{2}\sqrt {-2\,{a}^{2}+2\,\sqrt {{a }^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}}}}+4\,{{\rm e}^{1/2\,\sqrt { -2\,{a}^{2}+2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}t}} \sqrt {-2\,{a}^{2}+2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\, b}{\it \_C4}\,{a}^{2}-{\it \_C1}\, \left ( -2\,{a}^{2}-2\,\sqrt {{a}^{2 } \left ( {a}^{2}+4\,b \right ) }-4\,b \right ) ^{{\frac {3}{2}}}{{\rm e} ^{-{\frac {t}{2}\sqrt {-2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+4 \,b \right ) }-4\,b}}}}-4\,{{\rm e}^{-1/2\,\sqrt {-2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}t}}\sqrt {-2\,{a}^{2}-2\, \sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}{\it \_C1}\,{a}^{2}+ {\it \_C2}\, \left ( -2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b \right ) ^{{\frac {3}{2}}}{{\rm e}^{{\frac {t}{2}\sqrt {-2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}}}}+ 4\,{{\rm e}^{1/2\,\sqrt {-2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+ 4\,b \right ) }-4\,b}t}}\sqrt {-2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a} ^{2}+4\,b \right ) }-4\,b}{\it \_C2}\,{a}^{2}-{\it \_C3}\, \left ( -2\,{ a}^{2}+2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b \right ) ^{ {\frac {3}{2}}}{{\rm e}^{-{\frac {t}{2}\sqrt {-2\,{a}^{2}+2\,\sqrt {{a }^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}}}}-4\,{{\rm e}^{-1/2\,\sqrt {-2\,{a}^{2}+2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}t}} \sqrt {-2\,{a}^{2}+2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\, b}{\it \_C3}\,{a}^{2}+4\,{{\rm e}^{1/2\,\sqrt {-2\,{a}^{2}+2\,\sqrt {{ a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}t}}\sqrt {-2\,{a}^{2}+2\, \sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}{\it \_C4}\,b-4\,{ {\rm e}^{-1/2\,\sqrt {-2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\, b \right ) }-4\,b}t}}\sqrt {-2\,{a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2 }+4\,b \right ) }-4\,b}{\it \_C1}\,b+4\,{{\rm e}^{1/2\,\sqrt {-2\,{a}^{ 2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}t}}\sqrt {-2\, {a}^{2}-2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}{\it \_C2 }\,b-4\,{{\rm e}^{-1/2\,\sqrt {-2\,{a}^{2}+2\,\sqrt {{a}^{2} \left ( {a }^{2}+4\,b \right ) }-4\,b}t}}\sqrt {-2\,{a}^{2}+2\,\sqrt {{a}^{2} \left ( {a}^{2}+4\,b \right ) }-4\,b}{\it \_C3}\,b \right ) } \right \} \right \} \]