\[ \left \{x''(t)=c^2 x(t) \left (3 \cos ^2(a t+b)-1\right )+\frac {3}{2} c^2 y(t) \sin (2 a b t),y''(t)=\frac {3}{2} c^2 x(t) \sin (2 a b t)+c^2 y(t) \left (3 \sin ^2(a t+b)-1\right )\right \} \] ✗ Mathematica : cpu = 0.0287914 (sec), leaf count = 0
DSolve[{Derivative[2][x][t] == c^2*(-1 + 3*Cos[b + a*t]^2)*x[t] + (3*c^2*Sin[2*a*b*t]*y[t])/2, Derivative[2][y][t] == (3*c^2*Sin[2*a*b*t]*x[t])/2 + c^2*(-1 + 3*Sin[b + a*t]^2)*y[t]},{x[t], y[t]},t]
, could not solve
DSolve[{Derivative[2][x][t] == c^2*(-1 + 3*Cos[b + a*t]^2)*x[t] + (3*c^2*Sin[2*a*b*t]*y[t])/2, Derivative[2][y][t] == (3*c^2*Sin[2*a*b*t]*x[t])/2 + c^2*(-1 + 3*Sin[b + a*t]^2)*y[t]}, {x[t], y[t]}, t]
✗ Maple : cpu = 0. (sec), leaf count = 0
dsolve({diff(diff(x(t),t),t) = (3*cos(a*t+b)^2-1)*c^2*x(t)+3/2*c^2*y(t)*sin(2*a*t*b), diff(diff(y(t),t),t) = (3*sin(a*t+b)^2-1)*c^2*y(t)+3/2*c^2*x(t)*sin(2*a*t*b)})
, result contains DESol or ODESolStruc
\[\text {Expression too large to display}\]