;;########################################################################
;; mmrvis.lsp
;; visualization for multivariate multiple regression ViSta model object
;; new visualization written by Forrest W. Young Oct-Dec 1997
;; Copyright (c) 1997 by Forrest W. Young
;;########################################################################

(defmeth mmr-model-object-proto :visualize ()
  (if (not (eq current-object self)) (setcm self))
  (when (= 1 (send self :num-iv))
        (error "Visualization not possible when there is only one predictor. Use univariate Regression Analysis."))
  (let* ((cpoints (center (select (send self :data-matrix)
                                  (iseq (send self :nobs))
                                  (send self :iv))))
         (nobs (send self :nobs))
         (num-iv (send self :num-iv))
         ;(spin-dim (min 9 num-iv))
         (spin-dim num-iv)
         (npts nil)
         (mmrmod self)
         (rays (transpose (send self :beta)));(rays (send self :coefs))
         (nredun (send self :redundancy))
         (redun-rays (if nredun
          (matmult 
           (diagonal  (select (send self :redun-evals) (iseq nredun)))
           (transpose (select (send self :redun-coefs)
                              (iseq num-iv) (iseq nredun))))))
         (scaled-coefs (select (column-list rays) (iseq spin-dim)))
         (dimension-lengths nil)
         (scaled-coefs-mat nil)
         (vector-lengths nil)
         (spin-vector-ratio nil)
         (redlabels nil)
         (denom-redun nil)
         (scale-factor-redun nil)
         (scaled-redun-coefs nil)
         (ray-labels (select (send self :variables) (send self :dv)))
         (variables (column-list cpoints))
         (scale-type 'fixed)
         (point-labels (send self :labels))
         (residuals (send (select (send self :reg-models) 0) :residuals))
         (cooks (send (select (send self :reg-models) 0) :cooks-distances))
         (leverages (send (select (send self :reg-models) 0) :leverages))
         (y-hat (col (send self :scores) 0))
         (variable-labels (select (send self :variables) (send self :iv)))
         (scatter nil)
         (cutoff 7)
         (var-list (when (> num-iv cutoff)
                         (name-list variable-labels :show nil 
                                    :title "Predictor Variables" :menu nil)))
         (scatmat (when (<= num-iv cutoff)
                        (scatterplot-matrix variables :show nil
                            :scale-type nil :point-labels point-labels
                            :variable-labels variable-labels
                            :title "Predictor ScatMat")))
         (spin-plot
          (if (> num-iv 2)
              (spin-plot  variables :show nil 
                          :scale-type 'variable
                          :point-labels point-labels
                          :variable-labels variable-labels
                          :title "Predictor Spin BiPlot")
              (plot-points variables :show nil
                            :scale-type 'centroid-variable
                            :point-labels point-labels
                            :variable-labels variable-labels
                            :title "Predictor BiPlot")))
         (influence-plot 
          (plot-points y-hat cooks :show nil :title "Response Influence"))
         (list-obs (name-list point-labels :show nil 
                              :title "Point and Vector Labels" ))
         (reg-plot 
          (plot-points (list 1 2) (list 1 2) :show nil 
                       :title "Response Fit"))
         (resid-plot 
          (plot-points y-hat residuals :show nil 
                       :title "Response Residuals"))
         (sp (spread-plot 
              (matrix '(2 3) (list (if (> num-iv cutoff) 
                                       var-list scatmat) 
                                   spin-plot influence-plot
                                   list-obs reg-plot  resid-plot))))
         )

    (defmeth sp :spreadplot-help ()
      (plot-help-window (strcat "SpreadPlot Help"))
      (paste-plot-help (format nil "This is the SpreadPlot for Multivariate Multiple Regression Analysis. In this SpreadPlot the windows are linked by the data's observations and variables.~2%"))
      (paste-plot-help (format nil "The Scatterplot Matrix window, which is in the upper left corner, lets you choose which Predictor Variables are displayed in other windows. You can select a single Predictor Variable by clicking on a diagonal cell in this matrix, or you can select two Predictor Variables by clicking on an off-diagonal cell. You can also select several Predictor Variables by shift-clicking on several cells.~2%"))
(paste-plot-help (format nil "The Point and Vector Labels window, at the lower left, presents labels for Observations (points) and Response Variables. Selecting Observation labels will cause points in the other plots to be highlighted. Selecting Response Variable labels will cause vectors to be highlighted, and will change several of the plots to show information about the Response Variable.~2%"))
(paste-plot-help (format nil
"The points in the windows of this spreadplot are linked together. When you brush or click on them in one window, the corresponding points in other windows are also highlighted. The points are linked together because they represent the same observations in your data.~2%"))
      (show-plot-help)
      (call-next-method :skip t :flush nil))


    (send influence-plot :add-slot 'switch 1)
    
    (defmeth influence-plot :adjust-screen-point (i)
            (let* ((npoints (send self :num-points)))
              (send self :slot-value 'switch 
                    (* -1 (send self :slot-value 'switch)))
              (cond
                ((> npoints i) (call-next-method i))
                (t 
                 (when (and (< i (+ npoints (send current-model :num-dv)))
                            (> 0 (send self :slot-value 'switch)))
                       (send self :plot-details current-model (- i npoints) ))
                 ))))

    (defmeth influence-plot :plot-details (mob i) 
      (let* ((cooks 
              (send (select (send mob :reg-models) i) :cooks-distances))
             (y-hat (col (send mob :scores) i))
             (minx (min y-hat))
             (maxx (max y-hat))
             (miny (min cooks))
             (maxy (max cooks))
             (gnrx (get-nice-range minx maxx 5))
             (gnry (get-nice-range miny maxy 5))
             )
        (send self :range 0 (first gnrx) (second gnrx) :draw nil)
        (send self :range 1 (first gnry) (second gnry) :draw nil)
        (send self :x-axis t t (third gnrx) :draw nil)
        (send self :y-axis t t (third gnry) :draw nil)
        (send self :clear-points)
        (send self :add-points y-hat cooks
              :point-labels (send mob :labels) :draw nil)
        #+color(send self :use-color t)
        #+color(send self :point-color 
                     (iseq (send self :num-points)) 'blue)
        (send self :point-symbol
              (iseq (send self :num-points)) 'square)
        (send self :variable-label 0 
              (strcat "Predicted " 
                      (select (select (send mob :variables) 
                                      (send mob :dv)) i)))
        (send self :variable-label 1 "Cook's Distances")
        (send self :redraw)))
    
    (send influence-plot :plot-details self 0)
    (send influence-plot :title "Influence")
    (send influence-plot :plot-buttons :new-x nil :new-y nil)
    (send influence-plot :showing-labels t)
    (send influence-plot :linked t)
    (send influence-plot :mouse-mode 'brushing)

    (defmeth influence-plot :plot-help ()
      (plot-help-window (strcat "Help for " (send self :title)))
      (paste-plot-help (format nil 
"The influence plot is a regression diagnostic plot: It helps diagonse the stabililty of the regression analysis for the data being analyzed. Influence plots may be used to determine the influence of a particular observation on the regression parameter estimates. In particular, the plot shows the effect, on the values of one of the response variables, of removing an individual observation.~2%"))
(paste-plot-help (format nil
"You can control which response variable is shown in this plot by clicking on a response variable's name in the POINT AND VECTOR LABELS window. The response variable names appear at the end of the list of labels.~2%"))
      (paste-plot-help (format nil 
"The influence plot uses Cook's distance measure. This measure determines the influence of removing an observation by estimating the difference between the regression coefficients calculated when the observation is included in the analysis and when it is omitted from the analysis.~2%"))
      (paste-plot-help (format nil 
"A large Cook's distance suggests that the observation has a large influence on the calculation of the parameter estimates: Small changes in the observation will have relatively large effects on the parameter estimates. If such an observation is not reliable, then the model is also not reliable and we do not have stable estimates of the parameters.~2%"))
      (show-plot-help)
      )
    
    (send resid-plot :add-slot 'switch 1)
   
    (defmeth resid-plot :adjust-screen-point (i)
            (let* ((npoints (send self :num-points)))
              (send self :slot-value 'switch 
                    (* -1 (send self :slot-value 'switch)))
              (cond
                ((> npoints i) (call-next-method i))
                (t 
                 (when (and (< i (+ npoints (send current-model :num-dv)))
                            (> 0 (send self :slot-value 'switch)))
                       (send self :plot-details current-model (- i npoints) ))
                 ))))

    (defmeth resid-plot :plot-details (mob i) 
      (let* (
             (residuals (send (select (send mob :reg-models) i) :residuals))
             (y-hat (col (send mob :scores) i))
             (minx (min y-hat))
             (maxx (max y-hat))
             (maxy (max (abs residuals)))
             (miny (- maxy))
             (gnrx (get-nice-range minx maxx 5))
             (gnry (get-nice-range miny maxy 5))
             )
        (send self :range 0 (first gnrx) (second gnrx) :draw nil)
        (send self :range 1 (first gnry) (second gnry) :draw nil)
        (send self :x-axis t t (third gnrx) :draw nil)
        (send self :y-axis t t (third gnry) :draw nil)
        (send self :clear-points)
        (send self :add-points y-hat residuals
              :point-labels (send mob :labels) :draw nil)
        #+color(send self :use-color t)
        #+color(send self :point-color 
                     (iseq (send self :num-points)) 'blue)
        (send self :point-symbol
          (iseq (send self :num-points)) 'square)
        (send self :variable-label 0 
              (strcat "Predicted " 
                      (select (select (send mob :variables) 
                                      (send mob :dv)) i)))
        (send self :variable-label 1 "Residuals")
        (send self :abline 0 0)
        (send self :redraw)))
    
    (send resid-plot :plot-details self 0)
    (send resid-plot :title "Residuals")
    (send resid-plot :plot-buttons :new-x nil :new-y nil)
    (send resid-plot :showing-labels t)
    (send resid-plot :linked t)
    (send resid-plot :mouse-mode 'brushing)

    (defmeth resid-plot :plot-help ()
      (plot-help-window (strcat "Help for " (send self :title)))
      (paste-plot-help (format nil 
"The residuals plot is a plot, for one of the response variables, of the residuals versus the predicted values for the selected response variable.~2%"))
(paste-plot-help (format nil
"You can control which response variable is shown in this plot by clicking on a response variable's name in the POINT AND VECTOR LABELS window. The response variable names appear at the end of the list of labels.~2%"))
      (paste-plot-help (format nil 
"The residuals plot is a regression diagnostic plot: It helps diagonse the suitability of the assumptions underlying regression analysis to the data being analyzed. Residual plots may be used to detect nonnormal error distributions, constant error variance (heteroscedasticity), nonlinearity and outliers.~2%"))
      (paste-plot-help (format nil 
"NORMALITY: The points in the plot should be randomly distributed about the zero line. If they are not, then the assumption of normality has probably not been met.~2%"))
      (paste-plot-help (format nil 
"LINEARITY: Points that form a systematic pattern, such as a curve, suggest that the assumption of linearity has been violated.~2%"))
      (paste-plot-help  (format nil "HETEROSCADASTICITY: The variance of the residuals should be about the same for all values of the predicted response variable. If the variance changes systematically with the response variable, then the assumption of constant error variance has not been met.~2%"))
      (paste-plot-help  (format nil "OUTLIERS: Outliers may be identified by examining observations which have residuals that are much larger than the rest of the residual values.~2%"))
  (show-plot-help)
  )

;scatmat methods and details
    (when scatmat
          (send scatmat :add-mouse-mode 'focus-on-variables
                :title "Focus On Variables"
                :click :do-new-variable-focus
                :cursor 'finger)
          (send scatmat :mouse-mode 'focus-on-variables)
          (send scatmat :plot-buttons :new-x nil :new-y nil)
          #+color(send scatmat :use-color t)
          #+color(send scatmat :point-color 
                       (iseq (send scatmat :num-points)) 'blue)
          (send scatmat :point-symbol
                (iseq (send scatmat :num-points)) 'square)
          (send scatmat :linked t))

;var-list methods and details
    (when var-list
          (send var-list :fix-name-list)
          ;(send var-list :has-h-scroll (max (screen-size)))
          ;(send var-list :has-v-scroll (max (screen-size)))
          (send var-list :redraw)
          (send var-list :menu nil)
          (defmeth var-list :do-select-click (x y m1 m2)
            (call-next-method x y m1 m2)
            (let* ((cur-var  (send self :selection))
                   (nvar nil) (variable-labels nil)
                   (var-labs nil) (cur-data nil) )
              (when cur-var
                    (setf nvar (send self :num-points))
                    (setf variable-labels 
                          (send self :point-label (iseq nvar)))
                    (setf var-labs (select variable-labels cur-var))
                    (setf cur-data 
                          (map-elements #'send current-data 
                                        :variable var-labs))
                    (send sp :update-spreadplot 0 0  cur-var 
                          (list var-labs cur-data)))
              ))
          )

;spin-plot and scatterplot methods and details
    (setf scatter spin-plot)
    (send spin-plot :scale-type 'centroid-variable)
    (send spin-plot :linked t)
    (send spin-plot :showing-labels t)
    #+color(send scatter :use-color t)
    #+color(send spin-plot :point-color 
                 (iseq (send spin-plot :num-points)) 'blue)
    (when (> num-iv 2) (send spin-plot :depth-cuing nil))
    (send spin-plot   :point-symbol
          (iseq (send spin-plot :num-points)) 'square)
    (setf dimension-lengths 
          (mapcar #'second (send spin-plot :range (iseq spin-dim))))
    (setf scaled-coefs-mat 
          (matrix (list spin-dim (send self :num-dv)) 
                  (combine scaled-coefs)))
    (setf vector-lengths
          (sqrt (mapcar #'ssq (column-list scaled-coefs-mat))))
    (setf spin-vector-ratio (/ (min dimension-lengths) 
                               (max vector-lengths)))
    (setf scaled-coefs (* scaled-coefs spin-vector-ratio))
    
    (send spin-plot   :add-rays scaled-coefs 
          :ray-labels ray-labels :ray-color 'green)
    (when redun-rays
          (setf redlabels
                (mapcar #'(lambda (x) (format nil "Red~d" x))
                        (iseq nredun)))
          (setf scaled-redun-coefs (* redun-rays spin-vector-ratio))
          (setf scaled-coefs-mat 
                (matrix (list spin-dim nredun) (combine scaled-redun-coefs)))
          (setf vector-lengths
                (sqrt (mapcar #'ssq (column-list scaled-coefs-mat))))
          (setf spin-vector-ratio (/ (min dimension-lengths) 
                                     (max vector-lengths)))
          (setf scaled-redun-coefs 
                (* scaled-redun-coefs spin-vector-ratio))
          (send spin-plot :add-rays (column-list scaled-redun-coefs) 
                :ray-labels redlabels :ray-color 'red)
          (send spin-plot :add-rays (column-list (* -1 scaled-redun-coefs)) 
                :ray-labels redlabels :ray-color 'red))

;spin-plot unique methods and details

    (cond 
      ((> num-iv 2)

       (defmeth spin-plot :plot-help ()
         (plot-help-window (strcat "Help for " (send self :title)))
         (paste-plot-help (format nil "The help for the spinning biplot is divided into two sections. The first section is specific help for the biplot as used in multivariate regression. The second section is general help about using spinning plots.~2%HELP FOR MULTIVARIATE REGRESSION SPINNING BIPLOT~2%"))
         (paste-plot-help (format nil "A Spinning Biplot is an enhanced spinning 3-dimensional scatterplot that uses both points and vectors to represent structure. As used in Multivariate Regression Analysis, the axes of a biplot are predictor variables, the points represent observations, and the vectors represent response variables. In particular, the blue observation points are located according to the scores of the observations on the predictor variables, and the green response variable vectors are located to represent the coefficients of the predictor variables for the response.~2%"))
         (paste-plot-help (format nil "The relative location of the points can be interpreted. Points that are close together correspond to observations that have similar scores on the predictor variables displayed in the plot. To the extent that these predictors fit the response variables well, the points also correspond to observations that have similar values on the response variables.~2%"))
(paste-plot-help (format nil "Both the direction and length of the vectors can be interpreted. Vectors point away from the origin in some direction. Vectors that point in the same direction correspond to variables that have similar relationships to the predictors, and can be interpreted as having similar meaning in the context set by the data. Long vectors are more strongly related to the predictors being displayed than are short vectors. Long vectors are more important in interpreting the meaning of the predictors that they are long on.~2%"))
(when redun-rays
      (paste-plot-help (format nil "Finally, the red vectors represent the redundancy variables, which are the redundancy analysis model of your data. The first redundancy variable is labeled RED0. It is the linear combination of the predictor variables which has the highest average squared correlation with all of the response variables. It as the direction in the predictor variable space that maximizes maximize the variance accounted for in the response variables. The second redundancy variable (RED1) is similar to RED0, but is defined in the residual space that remains after computing RED0.~2%")))
      (paste-plot-help (format nil "HELP FOR SPINNING PLOTS~2%"))
  (show-plot-help)
  (call-next-method :flush nil))
       
       (send spin-plot :plot-buttons :margin nil :box t :new-x nil :new-y nil)
       (defmeth spin-plot :update-plotcell (i j args)
         (when (and (= i 0) (= j 0))
            (let* ((cur-var-nums (remove-duplicates (first args)))
                   (cur-var-names (remove-duplicates 
                                   (first (second args))  :test 'equal))
                   (numvars (send self :num-variables))
                   )
              (when (<= (length cur-var-nums) 2)
                    (setf cur-var-nums 
                          (select 
                           (combine cur-var-nums 
                            (set-difference (send self :current-variables)
                                            cur-var-nums))
                           (iseq 3)))
                    (setf cur-var-names 
                          (select (send self :variable-labels)
                                  cur-var-nums)))
              (when (or (= (length cur-var-nums) 3)
                        (and (= (length cur-var-nums) 4)
                             (= (third (send self :current-variables)) 
                                (- numvars 1))))
                    (when (= (length cur-var-nums) 4)
                          (setf cur-var-nums (select cur-var-nums '(0 1 2)))
                          (setf cur-var-names 
                                (select cur-var-names '(0 1 2)))
                          )
                    (apply #'send self  :current-variables cur-var-nums)
                    (send self :set-variables-with-labels cur-var-nums
                          cur-var-names)
                    (send self :transformation nil :draw nil)
                    (send self :add-box)
                    (when (matrixp (send self :slot-value 'rotation-type))
                          (send self :slot-value 'rotation-type 'yawing))
                    (send self :redraw)
                    ))))
       
       
       (send spin-plot :mouse-mode 'hand-rotate)
       
       (send spin-plot :set-variables-with-labels '(0 1 2)
             (select (send spin-plot :variable-labels) '(0 1 2))) 
       
       (defmeth spin-plot :add-box ()
         (call-next-method)
         (send spin-plot   :add-rays scaled-coefs :ray-color 'green)
         (when redun-rays
               (send spin-plot :add-rays (column-list scaled-redun-coefs) 
                     :ray-color 'red)
               (send spin-plot :add-rays 
                     (column-list (* -1 scaled-redun-coefs))
                     :ray-color 'red))
         (send self :redraw-content) )

       (send spin-plot :switch-add-box)       
       )
      (t

;scatterplot methods and details

       (send scatter :plot-buttons :new-x nil :new-y nil)

       (defmeth scatter :redraw ()
         (call-next-method)
         (send self :add-grid))
       
       (defmeth scatter :plot-details () 
         (let* ((x (send self :point-coordinate 0 (iseq nobs)))
                (y (send self :point-coordinate 1 (iseq nobs)))
                (maxx (max (abs x)))
                (minx (- maxx))
                (maxy (max (abs y)))
                (miny (- maxy))
                (gnrx (get-nice-range minx maxx 5))
                (gnry (get-nice-range miny maxy 5))
                (minaxx (first  gnrx))
                (maxaxx (second gnrx))
                (minaxy (first  gnry))
                (maxaxy (second gnry))
                )
           (send self :range 0 minaxx maxaxx :draw nil)
           (send self :range 1 minaxy maxaxy :draw nil)
           (send self :x-axis t t (third gnrx) :draw nil)
           (send self :y-axis t t (third gnry) :draw nil)
           (send self :add-grid)
           (send self :redraw)
           ))

       (send scatter :plot-details)

       ))

;reg-plot methods and details          
        
    (send reg-plot :add-slot 'switch 1)
    (send reg-plot :title "Fit Plot")
    
    (defmeth reg-plot :reg-plot (i mob) 
      (let* ((x (send (select (send mob :reg-models) i) :y))
             (y (col (send mob :scores) i))
             (min (min (combine x y)))
             (max (max (combine x y)))
             (gnr (get-nice-range min max 5))
             (axmin (first gnr))
             (axmax (second gnr))
             )
        (send self :range 0 (first gnr) (second gnr) :draw nil)
        (send self :range 1 (first gnr) (second gnr) :draw nil)
        (send self :x-axis t t (third gnr) :draw nil)
        (send self :y-axis t t (third gnr) :draw nil)
        
        (send self :clear)
        (send self :add-points x y 
              :point-labels (send mob :labels) :draw nil)
        #+color(send self :use-color t)
        #+color(send self :point-color 
                     (iseq (send self :num-points)) 'blue)
        (send self :point-symbol
              (iseq (send self :num-points)) 'square)
        (send self :variable-label 0
              (select (select (send mob :variables) 
                              (send mob :dv)) i))
        (send self :variable-label 1 
              (strcat "Predicted " 
                      (select (select (send mob :variables) 
                                      (send mob :dv)) i)))
        (send self :add-lines 
              (list (list axmin axmax) (list axmin axmax)) :draw nil)
        (send self :redraw)))
    
    (defmeth reg-plot :adjust-screen-point (i)
      (let* ((npoints (send self :num-points)))
        (send self :slot-value 'switch 
              (* -1 (send self :slot-value 'switch)))
        (cond
          ((> npoints i) (call-next-method i))
          (t 
           (when (and (< i (+ npoints (send current-model :num-dv)))
                      (> 0 (send self :slot-value 'switch)))
                 (send self :reg-plot (- i npoints) current-model))
           ))))
    (send reg-plot :reg-plot 0 self)
    (send reg-plot :showing-labels t)
    (send reg-plot :linked t)
    (send reg-plot :mouse-mode 'brushing)
    
    (send reg-plot :plot-buttons :new-x nil :new-y nil)
    
    (defmeth reg-plot :plot-help ()
      (plot-help-window (strcat "Help for " (send self :title)))
      (paste-plot-help (format nil 
"The Fit plot is a plot, for one of the response variables, of the predicted values of the response variable versus the observed value of the response variable.~2%"))
(paste-plot-help (format nil
"You can control which response variable is shown in this plot by clicking on a response variable's name in the POINT AND VECTOR LABELS window. The response variable names appear at the end of the list of labels.~2%"))
(paste-plot-help (format nil
"The line through the plot represents perfect fit. To the extent that the points deviate from this line the predicted values are not fitting the actual observed response variable well. The greater the deviation, the worse the fit.~2%"))
      (show-plot-help)
      )
    
;list-obs methods and details
    (send list-obs  :linked t)
    #+color(send list-obs :use-color t)
    #+color(send list-obs :point-color (iseq nobs) 'blue)
    (send list-obs :add-points (send self :num-dv) 
          :point-labels ray-labels)
    (send list-obs :point-color 
          (iseq nobs (+ nobs (- (send self :num-dv) 1))) 'green)
    (when redun-rays
          (setf npts (send list-obs :num-points))
          (send list-obs :add-points (send self :redundancy)
                :point-labels redlabels)
          (send list-obs :point-color 
                      (iseq npts 
                            (+ npts (- (send self :redundancy) 1))) 'red))
    (send list-obs :fix-name-list)
    
    (send sp :show-spreadplot)
    (send (send spin-plot :menu) :title "BiPlot")
    (send (send influence-plot :menu) :title "Influence")
    (send (send resid-plot :menu) :title "Residuals")
    (send (send reg-plot :menu) :title "RegPlot")
    (send (send list-obs :menu) :title "Labels")
  
    t))

(provide "mmrvis")