とりあえず関数名をjjjにしてみました

(define (square x) (* x x))
(define (sum-of-squares x y)
  (+ (square x) (square y)))

(define (jjj a b c) (if (> a b) (if (> b c) (sum-of-squares a b) (sum-of-squares a c)) (if (> a c) (sum-of-squares b a) (sum-of-squares b c)) ))

(jjj 1 2 3) (jjj 2 1 3) (jjj 3 2 1)