; Choose best images per direction
 ;
 ; SOURCE:      spider/docs/techs/recon1/Procs/best-imgs.spi      
 ;
 ; PURPOSE:     Limits particles from each reference direction  
 ;              Continue with reconstruction following this limitation using:
 ;              recon-regroup.spi
 ;
 ; USAGE:       spider spi/dat @best-imgs
 ;
 ; REQUIRES:    spider/docs/techs/recon1/Procs/verify-settings.spi
 ;
 ; INPUTS:  (Where ### denotes view   *** denotes group)
 ;  [ref_view_list]     [rec_dir]/sel_proj               Projection/view list       (one)
 ;  [sel_group]         [win_dir]/sel_group              Old group selection file   (one)
 ;  [sel_part]          [win_dir]/sel_part_***           Particle selection list    (one/group)
 ;  [good_particles]    views/prj###/sel_part_byv_good   Selection & CCC files (if using verify) (one/view)
 ;     or               views/prj###/sel_part_byv_goodB  Selection & CCC files (if using verify & recheck) (one/view)
 ;     or               views/prj###/sel_part_byv_sort   Selection files       (if not using verify) (one/view)
 ;
 ; OUTPUTS:  
 ;  [sel_group_best]    sel_group_best'                  New group selection file      (one)
 ;  [sel_part_best]     sel_part_best_***'               New particle selection files  (one/group)
 ;  [best_particles]    views/prj###/sel_part_byv_best'  Selection & CCC files         (one/view)
 ;  [parts_vsview_best] views/parts_vsview_best'         Total # of good particles     (one/view)

 ;  -------------- Parameters -------------------------------------

 [maxim] = -1                ; Max number of images/reference-view to retain (<0 == all)

 ; Type of verification used determines particle selection file name 
 [use-particles] = 0         ; Use:  (sorted particles == 0 ), (class verified == 1) (class verified & rechecked == 2)
 
 ;  -------------------- END BATCH HEADER --------------------

 MD
   TR OFF                                   ; Decrease results file output
 MD
   VB OFF                                   ; Decrease results file output

 ; Set common filenames & parameters
 @verify-settings

 ; Evaluate whether input suffix is: sorted, good,  or  goodB

 [particles] = [sorted_particles]           ; No classification verification

 IF ( [use-particles] == 1 ) THEN
    [particles] = [good_particles]          ; Used classification verification
 ELSEIF  ( [use-particles] == 2 ) THEN
    [particles] = [new_good_particles]      ; Used classification verification with recheck
 ENDIF


 sys
  echo "  Using: [particles]"  ; echo


 ; Set temporary files
 [sel_part_prev]           = '[sel_part]_prev'       ; Renamed old particle selection files (one/group)
 [temp_parts_byview]       = 'tmpdoc_byview_incore'
 [temp_sorted_byview]      = 'tmpdoc_byview_sorted'
 [temp_combined]           = 'tmpdoc_combined'
 [temp_grouplist_unsorted] = 'tmpdoc_bygrp_unsort{***[grp]}'

 ; Delete pre-existing files
 DE
   [sel_group_best]
 SD /      GROUP_NUM     AFTER_LIM     BEFORE_LIM     PERCENT
   [sel_group_best]
 DE
   [temp_combined]
 DE
   [parts_vsview_best]
 SD /      AFTER_LIM     BEFORELIM      PERCENT
   [parts_vsview_best]

 UD N [views]                       ; Get number of views
   [ref_view_list]                  ; Doc file                    (input)

 IF ( [maxim] > 0 ) THEN
   SYS
     echo -n '  Keeping up to: {%I0%[maxim]} particles from: {%I0%[views]} views    ' ; date
 else
   SYS
     echo -n '  Keeping all particles from: {%I0%[views]} views    ' ; date
 endif

 ; Initialize counters
 [totview-before] = 0
 [totview-after]  = 0
 [part-counter]   = 0                 ; File    (output) key counter
 [zero]           = 0
 [dum]            = 0

 DO [view]=1,[views]                  ; Loop over all views for this group

    UD N [view-parts]                 ; Get number of images for this view
      [particles][view]               ; Listing file                (input)

    IF ( [view-parts] <=  0 ) THEN
      ; Write to summary
      SD [view], [zero],[zero],[zero]
        [parts_vsview_best]           ; File      (output)
        
      ; Create empty file
      SYS
        touch [best_particles][view].$DATEXT
      ; (Downstream batch files may look for this file.)

      CYCLE                           ; Skip to next view
    ENDIF
    
    ; Initialize in-core stack
    SD IC NEW
      [temp_parts_byview]
      7, [view-parts]                 ; Incore file    (output) 
    
    ; Retain only best particles
    IF ( [view-parts] > [maxim] ) THEN

      IF ( [maxim] > 0 ) THEN
        ; Only keep maxim images

        DOC SORT
          [particles][view]            ; Particle listing file        (input)
          [temp_sorted_byview]         ; Temp sorted listing file     (output)
          -4                           ; Column to be sorted (CCROT)
          Y                            ; Compress & renumber keys

        DO [pkey2] = 1,[maxim]
          UD IC [pkey2], [view-slice2],[glo-num2],[grp-slice2],[ccrot],[mirror],[grp-num2],[view-num2]
            [temp_sorted_byview]       ; Listing file          (input)

          SD IC [pkey2], [view-slice2],[glo-num2],[grp-slice2],[ccrot],[mirror],[grp-num2],[view-num2]
            [temp_parts_byview]

          [part-counter] = [part-counter] + 1

          !SD [part-counter], [grp-slice2],[view-slice2],[grp-num2],[view-num2],[glo-num2]
          SD [part-counter], [dum],[dum],[dum],[glo-num2],[dum]
            [temp_combined]                 ; File    (output)
        ENDDO
            
        [percent-kept]   = ([maxim]/[view-parts])*100
        [totview-before] = [totview-before] + [view-parts]
        [totview-after]  = [totview-after] + [maxim]
            
        ; Write to summary
        SD [view], [maxim],[view-parts],[percent-kept]
          [parts_vsview_best]
            
        ; Close docs
        UD ICE
          [temp_sorted_byview]          ; Temp sorted listing file (closed)
        DE
          [temp_sorted_byview]

        GOTO LB1
      ENDIF
    ENDIF
            
    ; Retain all particles (if [maxim] <= 0 OR [view-parts] > [maxim])

    DO  [pkey6] = 1,[view-parts]
      UD IC [pkey6], [view-slice6],[glo-num6],[grp-slice6],[ccrot],[mirror],[grp-num6],[view-num6]
        [particles][view]             ; Listing file            (input)

      SD IC [pkey6], [view-slice6],[glo-num6],[grp-slice6],[ccrot],[mirror],[grp-num6],[view-num6]
        [temp_parts_byview]

      [part-counter] = [part-counter] + 1
      
      !SD [part-counter], [grp-slice6],[view-slice6],[grp-num6],[view-num6],[glo-num6]
      SD [part-counter], [dum],[dum],[dum],[glo-num6]
        [temp_combined]
        ; TO DO: check if DOC COMBINE + SD IC is faster
    ENDDO

    [percent-kept]   = 100
    [totview-before] = [totview-before] + [view-parts]
    [totview-after]  = [totview-after] + [view-parts]
    
    ; Write to summary
    SD [view], [view-parts],[view-parts],[percent-kept]
      [parts_vsview_best]
    
    ; Close docs
    UD ICE
      [particles][view]                ; Temp sorted listing file
    
  LB1

 
    ; Close docs
    SD IC COPY
      [temp_parts_byview]
      [best_particles][view]
    SD IC E
      [temp_parts_byview]
    SD /      VIEW_WIN     GLOBAL_NUM     GRP_WIN         CCROT        MIRROR        GRP_NUM       VIEW
      [best_particles][view]
    SD E
      [best_particles][view]
 ENDDO                    ; End view-loop -------------------------------------

 SYS
   echo "  Kept: {%I0%[totview-after]} Out of: {%I0%[totview-before]} particles In: {%I0%[views]} views"

 ; Close docs
 [dummy] = -[views]
 SD /           AFTER_LIM     BEFORELIM
   [parts_vsview_best]
 SD [dummy], [totview-after],[totview-before]
   [parts_vsview_best]
 SD E
   [parts_vsview_best]
 SD E
   [temp_combined]


 ; GENERATE GROUP PARTICLE DOCS xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

 ; Get # of groups
 UD N [num-grps]
   [sel_group]

 SYS
   echo ; echo -n "  Keeping good particles from: {%I0%[num-grps]}  groups    " ; echo
 
 MD
   VB ON

 ; Initialize counters
 [totgrp-before] = 0
 [totgrp-after]  = 0

 ; Loop through groups
 DO  [grp] = 1,[num-grps]        ; Loop through groups ---------------------------------

    ; Get # of particles before in this group
    UD N [grpparts-before]
      [sel_part][grp]
    
    ; Intersect group particle list with combined list
    DOC AND
      [sel_part][grp]            ; File    (input)
      [temp_combined]            ; File    (input)
      [temp_grouplist_unsorted]  ; File    (output)
      4                          ; Column # to intersect: global #
    
    ; Check if previous file exists
    IQ FI [sel-exists]
      [sel_part_best][grp]       ; File  (input)
    IF ( [sel-exists] == 1 ) THEN
      ; If it exists, rename
      SYS                        ; Rename existing selection file 
        \mv -v [sel_part_best][grp].$DATEXT   prev_[sel_part_best][grp].$DATEXT
    ENDIF

   !SD /      APSHSLICE     MIC_SLICE     APSHGROUP      MICNUM       GLOBALNUM
    SD /        WIN          MIC_NUM       MIC_WIN      GLOBAL_NUM        GRP
      [sel_part_best][grp]       ; File  (output)
    SD E
      [sel_part_best][grp]       ; File  (closed)

    ; Renumber keys & resort
    DOC SORT A
      [temp_grouplist_unsorted]  ; File    (input)
      [sel_part_best][grp]       ; File    (output)
      4                          ; Column # to sort by: global #
      Y                          ; Renumber keys
    
    ; Clean up
    DE
      [temp_grouplist_unsorted]  ; File  (removed)

    ; Get # particles after
    UD N [grpparts-after]
      [sel_part_best][grp]       ; File  (input)
    
    ; Increment counters
    [percent-kept]  = ([grpparts-after]/[grpparts-before])*100
    [totgrp-before] = [totgrp-before] + [grpparts-before]
    [totgrp-after]  = [totgrp-after]  + [grpparts-after]
    
    ; Write to summary
    SD [grp], [grp],[grpparts-after],[grpparts-before],[percent-kept]
      [sel_group_best]         ; File  (output)
    
    SYS
      echo "  Group: {***[grp]}: Kept: {%I0%[grpparts-after]} out of: {%I0%[grpparts-before]} particles"
 ENDDO                         ; End group-loop -------------------------------------

 SYS
   echo ; echo -n "  Done -- Assigned: {%I0%[totgrp-after]} out of: {%I0%[totgrp-before]}  particles into: {%I0%[num-grps]} groups   " ; date ; echo

 ; Close docs
 DE
   [temp_combined]            ; File  (removed)

 [dummy] = -[num-grps]
 SD /           AFTER_LIM     BEFORELIM
   [sel_group_best]         ; File  (output)
 SD [dummy], [totgrp-after],[totgrp-before]
   [sel_group_best]         ; File  (output)
 SD E
   [sel_group_best]         ; File  (closed)

 EN

 ; Modified 2016-03-25
 ;    2016-03-25 (agl) -- New file locations
 ;    2013-11-28 (trs) -- bug fixes
 ;    2013-10-23 (agl) -- New file names, modernized syntax & cosmetic
 ;    2012-05-01 (trs) -- now required but added option to keep all particles

 ;