; Restacks particles and optionally applies CTF correction
 ;
 ; PURPOSE: Restacks particles listed in a set of particle selection files 
 ;          into specified number of new stacks. Can apply CTF correction
 ;          filter to files while restacking.
 ;  
 ; SOURCE:  spider/docs/techs/recon1/Procs/stackwins-n-ctf.spi  
 ; 
 ; USAGE:   spider spi/dat @stackwins-n-ctf

 ; ------------------- Parameters -------------------

 [want-stats-only-yn] = 0      ; Want statistics only and no stacks? (1 == no stacks)

 [apply-ctf-yn]       = 1      ; Apply ctf correction?      (0 == no)
                               ;  (Used to restack images that have not been corrected)

 [want-align-yn]      = 0      ; Want to copy alignment files also? (0 == no)
                               ;  (Used to duplicate an existing alignment file set)
 
 [want-global-yn]     = 1      ; Want global lookup table?  (0 == no)


 ; ------------------ Input files -----------------------

 [old_sel_grp]    = '../sel_micrograph'                   ; Group (or micrograph) selection file (one)

![old_sel_part]   = '../Particles/win/sel_part_****'      ; Particle selection files  (one/micrograph)
 [old_sel_part]   = '../Particles/win/sel_good_****'      ; Particle selection files  (one/micrograph)

 [old_stack]      = '../Particles/win/data_bymic_****'    ; Existing particle stacks  (one/micrograph)

 [ctf_cor]        = '../Power_Spectra/power/flipctf_****' ; OPTIONAL CTF correction file       (one/micrograph) 

![old_align]      = 'input/align_01_***'                  ; OPTIONAL Alignment parameter files (one/group)      
 [old_align]      = '*'                                   ; OPTIONAL Alignment parameter files             

 ; ----------------- Output files -----------------------
 ;
 ; Set in: recon-settings according to [steptype]
 ; OUTPUTS:  (Where [win_dir] denotes new input directory, [rec_dir] denotes recon directory,  ### denotes view   *** denotes group) 
 ; [sel_group]        [win_dir]/sel_group'       ; New group selection file      (one)
 ; [sel_parts]        [win_dir]/sel_part_***'    ; New particle selection files  (one/group) 
 ; [unaligned_images] [win_dir]/data_***'        ; New particle stacks           (one/group)
 ; [part2global]      [win_dir]/part2glonum_***' ; OPTIONAL Global particle lookup table  (one/group) 
 ; [global2grp_lut]   [win_dir]/global2group'    ; OPTIONAL Combined global lookup table  (one)      
 ; [next_group_align] [rec_dir]/align_01_***'    ; OPTIONAL Alignment parameter files     (one/group) 
 ;
 ; -------------- END BATCH HEADER -------------------------------

 MD                                 ; Skip unnecessary output 
   VB OFF
 MD                                 ; Skip unnecessary output 
   TR OFF

 ; Get reconstruction file names
 [steptype] = 0                     ; Output is for alignment step
 @recon-settings([steptype],[num-grps],[pixsiz],[ang-step],[r2],[alignsh],[prj-radius],[winsiz],[incore-yn],[bp-type],[qsub])
 [iter]     = 1

 SYS                                ; Create output directory if needed
   mkdir -p [win_dir]

 UD N [old_num_grps]                ; Find number of old groups (usually micrographs)
   [old_sel_grp]                    ; Old group (micrograph) selection file (input)

 [ntot] = 0                         ; Initialize overall cumulative total

 DO                                 ; Loop over all old groups (micrographs) ---------------
    UD NEXT [key], [ingrp]          ; Get old group (micrograph) number
      [old_sel_grp]                 ; Old group (micrograph) selection file (input)
    IF ( [key] <= 0 ) EXIT

    ; Accumulate total number of particles
    UD N [numparts]
      [old_sel_part][ingrp]         ; Old particle selection file     (input) 

    SYS
      echo "  Group: {%I4%[ingrp]}  has: {%I6%[numparts]} particles."

    ; Count number of selected particles
      [ntot] = [ntot] + [numparts]        
 ENDDO                             ; End loop over micrographs -----------------------------

 UD NEXT E                         ; Finished with doc file
   [old_sel_grp]                   ; Finished with doc file      (input)

 [newnpg]  = [ntot]/[num-grps]   ; Number of particles in new groups
 [newnpg1] = INT([newnpg])                         ; Number of particles in new groups
 IF ([newnpg] > [newnpg1])[newnpg]=INT([newnpg]+1) ; Number of particles in new groups

 SYS
   echo ; echo "  Restacking: {%I6%[ntot]} particles" 
 SYS
   echo "  From: {%I4%[old_num_grps]} groups into: {%I3%[num-grps]} groups, with {%I5%[newnpg]} particles each." ; echo

 IF ( [want-stats-only-yn] == 1 ) THEN
    EN
 ENDIF

 DE                                    ; Remove any new group selection file 
   [sel_group]                         ; New group selection file      (removed)
 SD /        GROUP       NUM_PARTS
   [sel_group]                         ; Group selection doc file      (output)
 [grp] = 1
 DE A
   [sel_parts]
 DE A                 
   [unaligned_images][grp]  
 DE A                 
   [part2global][grp]

 IF ( [want-global-yn] == 1 ) THEN
    DE
      [global2grp_lut]
    SD /      APSHSLICE     MIC_SLICE     APSHGROUP      MIC_NUM
      [global2grp_lut]
 ENDIF

 ; Compute number of particles for each new group
 [newnpg]  = [ntot]/[num-grps]                   ; Number of particles in new groups
 [newnpg1] = INT([newnpg])                         ; Number of particles in new groups
 IF ([newnpg] > [newnpg1])[newnpg]=INT([newnpg]+1) ; Number of particles in new groups

 [grp]       = 0                            ; Initialize output group number
 [outpart]      = [newnpg] + 1              ; To force new output group
 [needold]      = 1                         ; To force new input group
 [part_counter] = 0                         ; Initialize overall counter

 DO                                         ; Loop over all old particles ----------------
    IF ( [needold] >= 1 ) THEN 
      ; Start another input group (micrograph)

      IF ( [want-align-yn] > 0) THEN
        UD ICE                              ; Close doc file
         [old_align][ingrp]                 ; Old alignment doc file (closed)
      ENDIF

      UD NEXT [key], [ingrp]                ; Get next group (micrograph) number
        [old_sel_grp]                       ; Group (micrograph) selection file    (input)

      IF ([key] <= 0) EXIT                  ; Finished with all old groups

      [needold] = 0                         ; Flag to use old group
    ENDIF

    [calcgrpnum] = int([part_counter]*[num-grps]/[ntot])+1  ; The group number we *should* be on

    ;  IF ( [outpart] > [newnpg] ) THEN     ; (OUTPART is incremented after this step, so OUTPART+1 would get included)
    IF ( [calcgrpnum] > [grp] ) THEN 
      [part_counter]                        ; DIAGNOSTIC outpart newgroup

      ; Must start another output group

      IF ( [grp] > 0) THEN
        ; Not first output group, give details on last output group
        SYS
          echo "  Filled group: {%I3%[grp]} with: {%I6%[outpart]} particles."

        SD E                               ; Close doc file
          [sel_parts]                      ; Finished with this particle selection file (closed)
        SD E                               ; Close doc file
          [next_group_align]               ; Finished with this alignment file (closed)
        SD E                               ; Close doc file
          [part2global][grp]               ; Global number doc file     (closed)
        SD [grp],[grp],[outpart]           ; Place group # in group selection file
          [sel_group]                      ; Group selection file        (output)
      ENDIF

      [grp]  = [grp] + 1                   ; New output group #

      IF ( [grp] <= [num-grps] ) THEN
        ; Delete pre-existing files and label new doc files
        DE                                 ; Remove existing particle selection file 
          [sel_parts]                      ; Particle selection  file    (removed)
        SD /          #        ORIG_MICRO     MIC_WIN_#     GLOBAL_#       GRP
          [sel_parts]                      ; Particle selection file (output)

        DE                                 ; Remove existing output stack
          [unaligned_images][grp]          ; Particle stack file    (removed)            

        IF ( [want-global-yn] > 0) THEN
          DE                               ; Remove existing global lookup file 
            [part2global][grp]             ; Existing global number lookup file (removed)
          SD / KEY: WIN-NUM   REG: GLOBAL-NUM  MICROGRAPH; Label for new global number doc file
            [part2global][grp]             ; New global number doc file (output)
        ENDIF

        IF ( [want-align-yn] > 0) THEN
          DE                               ; Remove existing alignment file 
            [next_group_align]             ; Existing global alignment file (removed)
          SD /  KEY       PSI,    THE,    PHI,   REF#,    EXP#,  CUM.{ROT,   SX,    SY},  NPROJ,   DIFF,  CCROT,     ROT,     SX,     SY,   MIR-CC
            [next_group_align]             ; New global number doc file (output)
        ENDIF

      ENDIF

      [outpart] = 0                        ; Reset output particle # to zero
    ENDIF                                  ; For starting new group

    UD NEXT [key], [inpart],[d2],[mic],[glonum]  ; Get next input particle number
      [old_sel_part][ingrp]                ; Old particle selection file    (input)

    IF ( [key] <= 0) THEN
      [needold] = 1                        ; Flag for need to start next input group

      UD NEXT E  ; (Goes into infinite loop without this)
        [old_sel_part][ingrp]

      CYCLE                                ; Finished with this input group
    ENDIF

    ; Have old and new particle numbers now
    [outpart] = [outpart] + 1              ; New output particle # 
    [part_counter] = [part_counter] + 1

    IF ( [apply-ctf-yn] > 0) THEN
      ; Apply CTF correction to image and restack
      TF COR
        [old_stack][ingrp]@******[inpart]    ; Existing particle stack      (input)
        [ctf_cor][mic]                       ; CTF correction file          (input)
        [unaligned_images]@******[outpart]   ; CTF corrected particle stack (output)
  
    ELSE
      ; No CTF correction, just restacking
      CP
        [old_stack][ingrp]@******[inpart]    ; Existing particle stack  (input)
        [unaligned_images]@******[outpart]   ; New particle stack       (output)

    ENDIF

    ;   FI H [mic],[glonum]                  ; Query image header values
    ;      [old_stack][ingrp]@******[inpart] ; Existing particle stack  (input)
    ;      MIC,NUM
    ;
    ;   FI H [mic]                           ; Query image header values
    ;      [old_stack][ingrp]@******[inpart] ; Existing particle stack  (input)
    ;      MIC     

    SD [outpart], [outpart],[mic],[inpart],[glonum],[grp]  ; Save new particle data
      [sel_parts]                            ; Particle selection file   (output)

    IF ( [want-global-yn] > 0) THEN
      SD [outpart],[glonum],[mic]            ; Save lookup table
        [part2global][grp]                   ; New global number doc file (output)
            
      SD [glonum], [outpart],[inpart],[grp],[mic]
        [global2grp_lut]
    ENDIF

    IF ( [want-align-yn] > 0) THEN
      ; Get image dimension
      UD IC [inpart], [r1],[r2],[r3],[r4],[r5],[r6],[r7],[r8],[r9],[r10],[r11],[r12],[r13],[r14],[r15]
        [old_align][ingrp]                  ; Old alignment doc file     (input)
      SD [outpart],[r1],[r2],[r3],[r4],[outpart],[r6],[r7],[r8],[r9],[r10],[r11],[r12],[r13],[r14],[r15]
        [next_group_align]                  ; New alignment doc file     (output)
    ENDIF
 ENDDO                                      ; End loop over all old particles ----------------

 IF ( [outpart] > 0) THEN
    SYS
      echo "  Filled group: {%I3%[grp]} with: {%I6%[outpart]} particles."

    SD E                                    ; Close doc file
      [sel_parts]                           ; Output particle selection file (closed)
 ENDIF

 UD ICE                                     ; Close doc file
   [old_align][ingrp]                       ; Old alignment doc file    (closed)
 ; UD NEXT E                                ; Close doc file
 ;   [sel_group]                            ; Group selection doc file  (closed)
 UD NEXT E                                  ; Close doc file
   [old_sel_grp]                            ; Group selection doc file  (closed)

 [dummy] = -[num-grps]
 SD /          PARTSBEFORE   PARTS_AFTER
   [sel_group]
 SD [dummy], [ntot],[part_counter]
   [sel_group]
 SD E
   [sel_group]

 IF ( [want-global-yn] == 1 ) THEN
   SD E
     [global2grp_lut]
 ENDIF

 SYS
   echo 

 EN

 ; 
; Modified 2016-07-07 ; 2016-07-07 (agl) -- From: restack-n-ctf ; 2015-11-18 (agl) -- Use 'TF COR' not 'FD'!! ; 2014-08-13 (trs) -- Added combined global lookup ; 2013-11-29 (trs) -- Bug fix: went into infinite (?) loop if in-core files weren't closed ; 2013-11-28 (trs) -- Bug fix: now gets micrograph# and global particle number from OLD_SEL_PART instead of stack ; 2013-10-16 (agl) -- Modernized ; 2012-07-19 (trs) -- Creates output directory if needed ; 2012-05-01 (trs) -- Optionally writes global particle number to header position 42 ; 2012-04-05 (trs) -- Doesn't use output from pnums.spi anymore ; 2012-04-05 (trs) -- Adapted from listallparticles.spi ; 2010-09-02 (trs & ay) -- Trap for >999 micrographs for 'SD [dummy]' line ; 2009-10-16 (trs) -- Removed option for unstacked images ; 2008-10-03 (trs) -- Added output file part2global ; 2008-10-03 (trs) -- Assigns global particle number ; 2007-12-17 (trs) -- Optional parameter for maximum number of particles per micrograph ; 2007-10-10 (trs) -- Gets number of micrographs from sel_micrograph ;