; Combine particles from good classes
 ;
 ; SOURCE:    spider/docs/techs/recon1/Procs/verify-combine-classes.spi
 ;
 ; PURPOSE:   Combine particles from good classes
 ;
 ; USAGE:     clean ; ./spider spi/dat @verify-combine-classes
 ;
 ; REQUIRES:  spider/docs/techs/recon1/Procs/verify-settings.spi
 ;
 ; INPUTS:  (Where *** denotes group,  ### denotes view,   ??? denotes class)
 ;   [ref_view_list]       [rec_dir]/sel_proj                 List of projection views            (one)
 ;   [prj_dir]             views/prj###                       View sub-directories                (one/view)
 ;   [good_classes_list]   views/prj###/sel_class_good        Selected classes                    (one/view)
 ;   [class_doc]           views/prj###/sel_part_byclass_???  Particle selection by class         (one/class/view) 
 ;   [view_list]           views/prj###/sel_part_byv          Total particle selection files      (one/view)
 ;   [byhand_list]         views/prj###/sel_part_byhand       OPTIONAL hand-picked particle list  (one/class)
 ;   [first_good_part_doc] views/prj###/first_good_part       OPTIONAL for reference views        (one/view)
 ;
 ; OUTPUTS:  
 ;   [good_particles]      views/prj###/sel_part_byv_good     Good-particle list                  (one/view)
 ;   [bad_particles]       views/prj###/sel_part_byv_bad      Sorted bad-particle list            (one/view) 
 ;   [parts_vsview_good]   views/parts_vsview_good            Total # of good particles           (one/view)
 ;   [good_particle_copy]  ../Particles/good/sel_part_***     OPTIONAL Archive copy of good particle list  (one/view)
 ;
 ; ---------------------- Parameters  ----------------------

 [archive-yn] = 0     ; Copy files to Particles/good/... for archiving? (0 == no)

 ; ------------------- END BATCH HEADER -------------------

 MD
   TR OFF                                    ; Decrease results file output
 MD
   VB OFF                                    ; Decrease results file output

 ; Set common filenames & parameters
 @verify-settings

 ; Set temporary filenames

 [temp_goodbyview_wdupes_unsort_noccrot]  = 'tmp_goodbyview_1_combined'
 [temp_goodbyview_nodupes_unsort_noccrot] = 'tmp_goodbyview_2_wodupes'
 [temp_goodbyview_nodupes_unsorted_ccrot] = 'tmp_goodbyview_3_ccrot'
 [temp_badbyview_unsort]                  = 'tmp_badbyview_1_unsort'
 [temp_good_class]                        = 'tmp_good_class'

 SYS
   echo -n "  Combining good particles    "; date

 IF ( [archive-yn] == 0 ) THEN
   SYS
     echo "  Not archiving copy of selection files"    ; echo
 ELSE
   SYS
     echo "  Archiving copy of output selection files" ; echo
 ENDIF

 ; Prepare output-stats doc
 DE
   [parts_vsview_good]
 ;SD /     UNVERIFIED  GOOD_CLASSES     KEPT      PERCENTAGE
 SD /        #KEPT      GOODCLASSES    PREVERIFY      PERCENT
   [parts_vsview_good]

 ; Initialize total kept-particle counter
 [kept-parts] = 0

 ; Get number of reference-views
 UD N [num-refs]
   [ref_view_list]

 ; Loop through reference-views
 DO  [view-key] = 1,[num-refs]
 
  ; Get view #
   UD IC [view-key],[view]
     [ref_view_list]

   IF ( [archive-yn] .NE. 0 ) THEN
     SYS
       cp -f [good_particles][view].$DATEXT  [good_particle_copy][view].$DATEXT
   ENDIF

   ; CHECK IF THERE ARE ANY PARTICLES

   ; Look for selection file
   IQ FI [sel-exists]
     [view_list][view]

   ; If no selection file
   IF ( [sel-exists] == 0 ) THEN
     SYS
       echo "  View: {%I4%[view]}   No particles found "

     [tot-view-parts]   = 0
     [good-class-parts] = 0
     [good-parts]       = 0
     [fraction-kept]    = 0

     SYS
       touch [good_particles][view].$DATEXT
     ; Downstream procedures will look for this file

     GOTO LB7
   ENDIF


   ; Get total number of particles in current reference view
   UD N [tot-view-parts]
     [view_list][view]

   ; Check if good-class list exists
   IQ FI [goodclasses-exists]
     [good_classes_list][view]         ; File  (input)

   IF ( [goodclasses-exists] == 1 ) THEN
     ; Get number of good classes
     UD N [num-good-classes]
       [good_classes_list][view]
   ENDIF

   ; If good-class list doesn't exist or is empty
   IF ( [num-good-classes]*[goodclasses-exists] == 0 ) THEN
     SYS
       echo "  View: {%I0%[view]}   No good classes found"

     [good-class-parts] = 0
     [good-parts]       = 0
     [fraction-kept]    = 0

     SYS
       touch [good_particles][view].$DATEXT
     ; Downstream procedures will look for this file

     ; Copy total-particle list to bad-particle list (to be sorted)
     DOC REN
       [view_list][view]         ; File  (input)
       [temp_badbyview_unsort]   ; File  (output)

     GOTO LB4
   ENDIF


   ; JUGGLE PARTICLE-PICKING MODES

   ; Clean up pre-existing files
   DE
     [good_particles][view]
   DE
     [temp_goodbyview_wdupes_unsort_noccrot]


   ; Get first good-class number
   UD 1, [class-num]
    [good_classes_list][view]         ; File  (input)
   UD E

   ; Check if by-hand class doc exists
   IQ FI [byhand-exists]
    [byhand_list][view]{***[class-num]}  ; File  (input)


   ; PARTICLES PICKED BY HAND

   ; If by-hand class docs exist then
   IF ( [byhand-exists] == 1 ) THEN
     SYS
       echo "  View: {%I0%[view]}  Using hand-picked particle list"

     ; Make sure by-hand lists exist (will crash DOC COMBINE otherwise)

     ; Loop through good classes
     DO  [good-class-key5] = 1,[num-good-classes]  ; Loop through good classes -----

       ; Get class number
       UD IC [good-class-key5], [class-num]
         [good_classes_list][view]

       ; Check if by-hand list exists
       IQ FI [b-exists]
         [byhand_list][view]{***[class-num]}

       ; If by-hand list doesn't exist, create empty one
       IF ( [b-exists] == 0 ) THEN
         SYS
           echo "  View: {%I4%[view]}, Class: {%I3%[class-num]}  No by-hand particle-list found -- Continuing"
         SYS
           touch [byhand_list][view]{***[class-num]}.$DATEXT
       ENDIF
     ENDDO                               ; End class-loop ----------------------

     ; Combine good classes for hand-picked particles
     DOC COMBINE
       [byhand_list][view]***                    ; File (input)
       [good_classes_list][view]                 ; Selection file (input)
       [temp_goodbyview_wdupes_unsort_noccrot]   ; File   (output)

     ; Remove duplicates & renumber
     AT IT
       [temp_goodbyview_wdupes_unsort_noccrot]   ; File   (input)
       [temp_goodbyview_nodupes_unsort_noccrot]  ; File   (output)

     ; Get # particles kept
     UD N [good-parts]
       [temp_goodbyview_nodupes_unsort_noccrot]  ; File   (input)

     ; Calculate fraction kept
     [fraction-kept] = [good-parts]/[tot-view-parts]

     ; Combine good classes for total particles
     DOC COMBINE
       [class_doc][view]                        ; File  template (input)
       [good_classes_list][view]                ; Selection file (input)
       [good_particles][view]_notbyhand         ; Temporary file (output)

     ; Renumber
     DOC RENUM
       [good_particles][view]_notbyhand         ; Temporary file (input)
       [good_particles][view]_nohandrenum       ; Temporary file (output)

     ; Get number of particles in good classes
     UD N [good-class-parts]
       [good_particles][view]_nohandrenum       ; Temporary   (input)

     ; Clean up
     UD ICE
       [good_classes_list][view]                ; File        (closed)
     DE
       [good_particles][view]_notbyhand         ; File        (removed)
     DE
       [good_particles][view]_nohandrenum       ; File        (removed)

     GOTO LB6
   ENDIF
   ; End by-hand conditional


   ; FIRST GOOD PARTICLE PICKED FOR EACH CLASS

   ; Check if first good particle doc exists
   IQ FI  [exits]
     [first_good_part_doc][view]

   ; If first good particle doc exists then
   IF ( [exits] == 1 ) THEN
      SYS
        echo "  View: {***[view]}  Starting from first good particle"

      [good-parts]       = 0
      [good-class-parts] = 0

      ; Loop through good classes
      DO  [good-class-key2] = 1,[num-good-classes]
        ; Get class number
        UD IC [good-class-key2],[class-num]
          [good_classes_list][view]

        ; Get number of particles in class
        UD N [class-parts]
          [class_doc][view][class-num]           ; File    (input)

        ; Get first good particle# from current class
        UD IC [class-num],[first-good-part]
          [first_good_part_doc][view]            ; File    (input)

        ; Increment good-class particle-counter
        [good-class-parts] = [good-class-parts] + [class-parts]
            
        ; Initialize flag that good particles haven't started
        [good-part-flag] = 0        

        ; Loop through particles
        DO [part-key] = 1,[class-parts]       ; Loop through particles ----------

          ; Get particle#
          UD IC [part-key],[part-num]
            [class_doc][view][class-num]      ; File    (input)
          ; PART-NUM is probably VIEW-SLICE, but it could be unstacked particle #

          ; Check if first good particle hasn't been reached
          IF ( [good-part-flag] == 0 ) THEN
            ; Check if current particle is first good one
            IF ( [part-num] == [first-good-part] ) [good-part-flag] = 1  
          ENDIF

          ; Check if first good particle has been reached
          IF ( [good-part-flag] == 1 ) THEN  
            ; Increment good-particle counter
            [good-parts] = [good-parts] + 1  

            ; Write particle # to truncated-class, good-particle list
            SD [good-parts],[part-num]
              [temp_goodbyview_wdupes_unsort_noccrot]   ; File    (output)
          ENDIF
        ENDDO                    ; End particle-loop ---------------------

        UD ICE
         [class_doc][view][class-num]              ; File    (closed)
      ENDDO                      ; End class-loop-------------------------

      SD E  ;                       Close doc
        [temp_goodbyview_wdupes_unsort_noccrot]    ; File    (closed)

      ; Remove duplicates
      AT IT
        [temp_goodbyview_wdupes_unsort_noccrot]    ; File    (input)
        [temp_goodbyview_nodupes_unsort_noccrot]   ; File    (output)
     ; End first good conditional


     ; WHOLE CLASSES PICKED

   ELSE
     ; Check to see if all selected classes actually exist 
     DE
       [temp_good_class]                       ; Temp selection file   (removed)
     [n] = 0
     DO 
       UD NEXT [key],[classnum]                ; Get class number
         [good_classes_list][view]             ; Selection file   (input)
       IF ( [key] == 0 ) EXIT                  ; End of list

       IQ FI [c-exists]                        ; See if class file exists
         [class_doc][view][classnum]           ; File template    (input)
       IF ( [c-exists] > 0) THEN
         [n] = [n] + 1                         ; This class exists
         SD [n], [classnum]                    ; Save class number in temp file
           [temp_good_class]                   ; Temp selection file   (output)
       ENDIF
     ENDDO
     SD E                                      ; Close temp file 
        [temp_good_class]                      ; Temp selection file   (closed)
     UD N [numlists]                           ; Get number of existing classes
        [temp_good_class]                      ; Temp selection file   (output)

     SYS
       echo "  View: {%I4%[view]}   Combining: {%I3%[numlists]} class-membership lists"

     ; Combine good classes
     DOC COMBINE
       [class_doc][view]                          ; File template    (input)
       [temp_good_class] ;[good_classes_list][view]                  ; Selection file   (input)
       [temp_goodbyview_wdupes_unsort_noccrot]    ; File             (output)

     ; Remove duplicates
     AT IT
       [temp_goodbyview_wdupes_unsort_noccrot]    ; File    (input)
       [temp_goodbyview_nodupes_unsort_noccrot]   ; File    (output)

     ; Get number of particles kept
     UD N [good-parts]
       [temp_goodbyview_nodupes_unsort_noccrot]   ; File    (input)

       [good-class-parts] = [good-parts]
   ENDIF
   ; End whole-classes conditional


   LB6  ; Jump here if picking by hand

   ; Clean up
   UD ICE
     [good_classes_list][view]
   UD ICE
     [first_good_part_doc][view]
   DE
     [temp_goodbyview_wdupes_unsort_noccrot]

   ; Add CCC info to selection-doc
   DOC AND
     [view_list][view]                          ; Re-keyed total particle selection file  (input)
     [temp_goodbyview_nodupes_unsort_noccrot]   ; File    (input)
     [temp_goodbyview_nodupes_unsorted_ccrot]   ; File    (output)
     1                                          ; Column # to find intersection

   DE
     [good_particles][view]         ; File    (removed)
   SD /     VIEW_WIN     GLOBAL_NUM    GRP_WIN        CCROT       MIRROR       GRP_NUM    VIEW
     [good_particles][view]         ; File    (output)
   SD E
     [good_particles][view]         ; File    (closed)

   ; Sort good particles by CCC, Append to labeled header
   DOC SORT A
     [temp_goodbyview_nodupes_unsorted_ccrot]
     [good_particles][view]         ; File      (output)
     -4                             ; Column # to sort: CCROT, Reverse sort order
     Y                              ; Renumber keys 

   ; Clean up
   DE
     [temp_goodbyview_nodupes_unsort_noccrot]         ; File    (removed)
   DE
     [temp_goodbyview_nodupes_unsorted_ccrot]         ; File    (removed)

   ; Copy to single directory (for archival purposes)
   IF ( [archive-yn] .NE. 0 ) THEN
     SYS
       cp -f [good_particles][view].$DATEXT [good_particle_copy][view].$DATEXT
   ENDIF


   ; CREATED SORTED BAD-PARTICLE LIST

   ; Create bad-particle list
   DOC SUBTRACT
     [view_list][view]          ; Re-keyed total-particle selection doc (w/CCC) (input)
     [good_particles][view]     ; Good-particle selection doc, no CCC           (input)
     [temp_badbyview_unsort]    ; Bad-particle selection doc, unrenumbered      (output)
     1                          ; Column # to subtract: view-slice#

   LB4  ; Jump here if no good-class list found

   DE
     [bad_particles][view]
   
   ; Get # of bad particles
   UD N [bad-parts]
     [temp_badbyview_unsort]

   IF ( [bad-parts] > 0 ) THEN
     ; Sort bad particles by CCC, from highest to lowest
     SD /     VIEW_WIN     GLOBAL_NUM    GRP_WIN        CCROT       MIRROR       GRP_NUM    VIEW
       [bad_particles][view]   ; File    (output)
     SD E
       [bad_particles][view]   ; File    (closed)

     DOC SORT A
       [temp_badbyview_unsort]
       [bad_particles][view]   ; File      (output)
       -4                      ; Column# to sort: CCROT, Reverse sort
       Y                       ; Renumber keys
   ENDIF

   ; Clean up
   DE
     [temp_badbyview_unsort]

 ; LB4  ; Jump here if no good-class list found

    ; Calculate percentage kept
   IF ( [tot-view-parts] > 0 )  [fraction-kept] = 100*[good-parts]/[tot-view-parts]


   LB7  ; Jump here if no selection file 

   ; Write to stats doc
 ; SD [view], [tot-view-parts],[good-class-parts],[good-parts],[fraction-kept]
   SD [view], [good-parts],[good-class-parts],[tot-view-parts],[fraction-kept]
     [parts_vsview_good]

   ; Increment total kept-particle counter
   [kept-parts] = [kept-parts] + [good-parts]
 ENDDO                       ; End view-loop -----------------

 ; Close docs
 UD ICE
   [ref_view_list]
 DE
   [temp_good_class]       ; Temp selection file   (removed)

 [dummy] = -[num-refs]     ; Dummy variable
 SD /         GOODPARTICLES
   [parts_vsview_good]     ; File    (output)
 SD [dummy], [kept-parts]  ; File    (output)
   [parts_vsview_good]
 SD E
   [parts_vsview_good]     ; File    (closed)

 SYS
   echo ; echo -n "  Done, kept: {%I0%[kept-parts]} particles    "; date ; echo

 EN ;D

 ; Modified 2013-10-23
 ;    2013-10-23 (agl) -- New file names, modernized syntax & cosmetic
 ;    2013-10-23 (agl) -- Replaced: reversedoc-7col with reverse DOC SORT
 ;    2012-03-15 (trs) -- Summary doc now has the format of HOW_MANY
 ;    2012-03-01 (trs) -- Switched to named registers
 ;    2011-01-11 (trs) -- Error-check for zero bad particles
 ;    2009-06-30 (trs) -- Creates bad-particle list even if no good particles
 ;    2009-06-05 (trs) -- Removes duplicate particles
 ;    2009-06-03 (trs) -- Changed format of select/sel files
 ;    2009-05-13 (trs) -- Sorted good and bad particle lists
 ;    2009-05-08 (trs) -- Changed fraction kept per view to percentage kept
 ;    2009-04-03 (trs) -- Added copy of GOODSEL for archival purposes (all in one directory)
 ;    2009-02-24 (trs) -- Selection files now written to select/prj###
 ;    2009-02-23 (trs) -- Added view# to GOODSEL output
 ;    2008-02-05 (trs) -- Added divide-by-0 trap for empty views
 ;    2007-10-05 (trs) -- Modified for stacked particles
 ;                     -- TO DO: check whether this still works on unstacked particles
 ;    2006-02-06 (trs,pp) -- Bug fix: if picking by hand, CCC wasn't added to GOOD_PARTICLE_DOC
 ;    2005-08-30 (trs,hg) -- Bug fix: error when good-class list not found
 ;    2004-10-19 (trs) -- Bug fix: I had labelled loops with LB2 twice
 ;    2004-07-23 (trs,ga) -- Added option to pick particles from classes by hand
 ;    2004-07-23 (trs) -- Gets number of total particles from SELECT/SEL rather than HOW_MANY
 ;    2004-05-31 (trs) -- Adds CCC to output selection doc
 ;    2004-05-05 (trs) -- Handles reference-views with no good particles
 ;    2004-05-04 (trs) -- Gets number of classes from class-stats doc

 ;