/*------------------------- New Concepts Consulting ---------------------------- - Application Module Name : OS/2 - File Name : rsync_backup.cmd - Date : June 06, 2018 - Author : Dariusz Piatkowski - - Description: - This script implements a SOURCE storage to TARGET storage folder routine - utilizing the RSYNC utility. It supports the notion of REMOTE or LOCAL folder - destinations. The difference is due to potentially different levels of the - OS/2 extended attribute support by either platform. - - There are a couple of modes of execution: - 1) the SOURCE and/or TARGET folders can be any location, LOCAL or a NETWORK - drive, however if it is a NETWORK drive it must be mapped to the local - filesystem as a drive, for example through a NetDrive mapping - 2) the backup_type parameter is of a LOCAL or REMOTE type, the following are - the assumptions about what filesystem attributes are supported on each: - a) LOCAL - this supports the FULL OS/2 attributes - b) REMOTE - this does not support the FULL OS/2 attributes (extended, etc.) - - Subsequently the rsync parameters are written accordingly. - - Parameters: - Usage => RSYNC_BACKUP (LOCAL/REMOTE) "SOURCE" "TARGET" - - Modification History - - REVISON DATE/TIME AUTHOR - - Rev 1.0.0 JUN 06, 2020 Dariusz Piatkowski - First version of the code. - - Rev 1.1.0 SEP 14, 2020 Dariusz Piatkowski - Re-defined the structure of the code so that the parameter handling is no - longer static. This approach is discussed in the OS2World forum, see - thread => https://www.os2world.com/forum/index.php/topic,2571.0.html - - Rev 1.1.1 SEP 20, 2020 Dariusz Piatkowski - Added CLI parameter processing handling as per Andreas Schnellbacher's - sample code. Much appreciated!!! - - Rev 1.1.2 SEP 25, 2020 Dariusz Piatkowski - Added the LOG and DEBUG procedures to better handle logging and debug tasks - - Rev 1.1.3 OCT 12, 2020 Dariusz Piatkowski - Changed the RSYNC parameter logic from a flat string to an array driven - build. Also included extra RYSNC variable pointing to the RSYNC.EXE path, - as well as optional use-defined LOG, EXCLUDE and PASSWORD filenames - - Rev 1.1.4 OCT 13, 2020 Dariusz Piatkowski - Took the final part of the script which actually starts the RSYNC run and - moved to a separate subroutine. This keeps the core part (main:) a bit - cleaner. - - Rev 1.1.5 OCT 14, 2020 Dariusz Piatkowski - Small addition: included the ability for the user to specify a one-off - RYSNC parameter that is NOT stored in the ARRAY. By default this is NULL, - however the program logic needed to process this is provided, should that - approach be eventually used. - ------------------------------------------------------------------------------*/ CALL RxFuncAdd 'SysLoadFuncs', 'RexxUtil', 'SysLoadFuncs' CALL SysLoadFuncs main: /* setup the GLOBAL variables */ debug_flag =0 calling_code ='MAIN' msg_level =0 msg_text ='Initializing...' crlf ='0D0A'x /* carriage return / line feed */ source_folder ="" /* SOURCE folder */ target_folder ="" /* TARGET folder */ source_folder_ok ="NO" /* SOURCE path verify flag */ target_folder_ok ="NO" /* TARGET path verify flag */ /* define the two arrays that store the RSYNC parameters */ rsync_param_local. = "" rsync_param_remote. = "" /* setup the RSYNC path, logging and control file variables */ rsync_exe_file ='G:\usr\bin\rsync.exe' /* location of the RYSNC EXE */ rsync_exclude_file ='G:\mptn\etc\rsync_exclude' /* EXCLUDE objects */ rsync_password_file ='' /* not used right now */ rsync_log_file_path ='G:\tmp\log\' /* location of the LOG file */ rsync_log_file_name ='' /* optional, if blank auto-generated */ rsync_cli ='' /* optional RSYNC parameter */ rsync_user_param ='' /* optional one-off user defined RSYNC parameter */ /* pull in the CLI params */ PARSE ARG cli_arg CALL debug debug_flag calling_code msg_level msg_text msg_text="CLI input params : >"||cli_arg||"<" CALL debug debug_flag calling_code msg_level msg_text /* parse the CLI params to pick up the BACKUP_TYPE, SOURCE and TARGET info */ CALL cli_parm_parse(cli_arg) /* put the BACKUP_TYPE paramter into UPPERCASE */ backup_type = translate(backup_type) /* check to see if a backup_type was specified */ IF backup_type = '' THEN backup_type = 'UNKNOWN' msg_text ="Starting RSYNC for LOCAL to "||backup_type||" backup!" CALL log debug_flag calling_code msg_level msg_text /* msg_text="SOURCE_FOLDER => "||source_folder - CALL log debug_flag calling_code msg_level msg_text - - msg_text="TARGET_FOLDER => "||target_folder - CALL log debug_flag calling_code msg_level msg_text */ /* check to see if we have all the required input params */ IF (backup_type = '') | (source_folder = '') | (target_folder = '') THEN DO msg_text=crlf||"ERROR => Missing BACKUP_TYPE, SOURCE_FOLDER and/or TARGET_FOLDER locations!" CALL log debug_flag calling_code msg_level msg_text msg_text=crlf||"USAGE => RSYNC_BACKUP (LOCAL/REMOTE) SOURCE_FOLDER TARGET_FOLDER" CALL log debug_flag calling_code msg_level msg_text RETURN END ELSE DO /* since the INPUT params are not NULL, verify that they can be accessed */ CALL verify_folders /* check the results and post appropriate message if problems found */ IF source_folder_ok <> "YES" THEN DO /* ran into a problem accessing the SOURCE folder */ msg_text=crlf||"Problem accessing the SOURCE folder!" CALL log debug_flag calling_code msg_level msg_text END IF target_folder_ok <> "YES" THEN DO /* ran into a problem accessing the TARGET folder */ msg_text=crlf||"Problem accessing the TARGET folder!" CALL log debug_flag calling_code msg_level msg_text END msg_text="source_folder_ok flag = "||source_folder_ok||", "||, "target_folder_ok flag = "||target_folder_ok CALL debug debug_flag calling_code msg_level msg_text /* if either one failed post the EARLY TERMINATION warning */ IF (source_folder_ok <> "YES") | (target_folder_ok <> "YES") THEN DO msg_text=crlf||"Terminating the program..." CALL log debug_flag calling_code msg_level msg_text RETURN END /* build the two arrays with LOCAL and REMOTE RSYNC parameters */ CALL build_rsync_parameter_arrays /* next few lines will build the RSYNC option parameters string, it - should look something like this: - "'--dry-run --archive --delete --progress --stats --whole-file - --no-xattrs --human-readable --quiet --exclude-from='|| - rsync_exclude_file||' --log-file='||rsync_log_file - Enquote(source_folder) Enquote(target_folder)" */ /* build the RSYNC CLI option parameters */ CALL build_rsync_options /* check to see if we have an EXCLUDE filename */ IF rsync_exclude_file <> '' THEN rsync_cli=rsync_cli||' --exclude-from='||rsync_exclude_file /* check to see if we have a user specified LOG filename */ IF rsync_log_file_name <> '' THEN /* use the user provided name */ rsync_log_file = rsync_log_file_path||rsync_log_file_name ELSE /* let's make the fully qualified RSYNC LOG filename */ rsync_log_file = rsync_log_file_path||'rsync_'||backup_type||'_backup.log ' /* check to see if we have a user defined one-off RYSNC parameter */ IF rsync_user_param <>'' THEN DO /* toss an entry into the DEBUG log */ msg_text="Detected a user defined one-off RYSNC parameter : "||rsync_user_param CALL debug debug_flag calling_code msg_level msg_text rsync_cli=rsync_cli||rsync_user_param END /* add the LOG filename to the RSYNC CLI parameter */ rsync_cli=rsync_cli||' --log-file='||rsync_log_file /* now finally include the SOURCE and TARGET locations */ rsync_cli=rsync_cli||Enquote(source_folder)||" "||Enquote(target_folder) /* alright, all the pre-work is done so go ahead and run RSYNC */ CALL rsync_execute /* put a final TASK COMPLETED msg into the DEBUG log */ msg_text="Execution COMPLETED successfully!"||crlf CALL debug debug_flag calling_code msg_level msg_text END EXIT 0 /* MAIN end */ /* subroutine to execute RSYNC */ rsync_execute: procedure expose debug_flag msg_level source_folder target_folder, rsync_cli rsync_log_file rsync_exe_file, backup_type crlf /* setup the variables */ calling_code ='rsync_execute' msg_level =msg_level+1 msg_text ='Initializing...' /* put a proc ENTRY into DEBUG */ CALL debug debug_flag calling_code msg_level msg_text /* update the RSYNC LOG file with NEW run stamp */ msg_text=crlf||"======================================================"||, "=========================" CALL lineout rsync_log_file,msg_text CALL log debug_flag calling_code msg_level msg_text /* get the START message built */ datetime_stamp = date('U')||" - "||time('N') msg_text = "RSYNC started: "||datetime_stamp||"..." CALL lineout rsync_log_file,msg_text CALL log debug_flag calling_code msg_level msg_text msg_text=crlf||"SOURCE_FOLDER => "||source_folder CALL lineout rsync_log_file,msg_text CALL log debug_flag calling_code msg_level msg_text msg_text="TARGET_FOLDER => "||target_folder CALL lineout rsync_log_file,msg_text CALL log debug_flag calling_code msg_level msg_text msg_text=crlf||"RSYNC CLI =>"||RSYNC_CLI CALL lineout rsync_log_file,msg_text CALL debug debug_flag calling_code msg_level msg_text msg_text=crlf||"-----------------------------------------" CALL lineout rsync_log_file,msg_text CALL debug debug_flag calling_code msg_level msg_text CALL lineout rsync_log_file /* reset the timer to 0, we will use later to get TOTAL duration */ CALL time('r') /* check to see if we have a user supplier RSYNC path */ IF rsync_exe_file <> '' THEN '@'||rsync_exe_file||' '||rsync_cli ELSE /* if not, rely on RSYNC being somewhere in the PATH */ '@'||'rsync'||' '||rsync_cli /* update the RSYNC LOG file with NEW run stamp */ msg_text="-----------------------------------------" CALL lineout rsync_log_file,msg_text CALL debug debug_flag calling_code msg_level msg_text /* get the FINISH message built */ datetime_stamp = date('U')||" - "||time('N') msg_text = crlf||"RSYNC completed: "||datetime_stamp||"..." CALL lineout rsync_log_file,msg_text CALL log debug_flag calling_code msg_level msg_text msg_text=crlf||"RSYNC took "||time('e')||" seconds to complete the "||, backup_type||" backup task!" CALL lineout rsync_log_file,msg_text CALL log debug_flag calling_code msg_level msg_text msg_text="============================================================"||, "===================" CALL lineout rsync_log_file,msg_text CALL log debug_flag calling_code msg_level msg_text CALL lineout rsync_log_file /* put a proc EXIT into DEBUG */ msg_text='Exiting...' CALL debug debug_flag calling_code msg_level msg_text /* reset DEBUG LEVEL tracker */ msg_level=msg_level-1 return 0 /* rsync_execute END */ /* subroutine to verify the SOURCE and TARGET paths */ verify_folders: procedure expose debug_flag msg_level source_folder target_folder, crlf source_folder_ok target_folder_ok /* setup the variables */ calling_code ='verify_paths' msg_level =msg_level+1 msg_text ='Initializing...' /* put a proc ENTRY into DEBUG */ CALL debug debug_flag calling_code msg_level msg_text /* call SysFileTree from RexxUtil library to check the valid SOURCE path */ msg_text="Validating the SOURCE folder path : "||source_folder CALL debug debug_flag calling_code msg_level msg_text CALL SysFileTree source_folder, 'found1.', 'DO' msg_text="Found "||found1.0||" instances of this folder" CALL debug debug_flag calling_code msg_level msg_text DO i = 1 TO found1.0 msg_text="Found SOURCE folder : "||found1.i CALL debug debug_flag calling_code msg_level msg_text /* set the flag accordingly */ source_folder_ok="YES" END /* call SysFileTree from RexxUtil library to check the valid TARGET path */ msg_text="Validating the TARGET folder path : "||target_folder CALL debug debug_flag calling_code msg_level msg_text CALL SysFileTree target_folder, 'found1.', 'DO' msg_text="Found "||found1.0||" instances of this folder" CALL debug debug_flag calling_code msg_level msg_text DO i = 1 TO found1.0 msg_text="Found TARGET path : "||found1.i CALL debug debug_flag calling_code msg_level msg_text /* set the flag accordingly */ target_folder_ok="YES" END /* put a proc EXIT into DEBUG */ msg_text='Exiting...' CALL debug debug_flag calling_code msg_level msg_text /* reset DEBUG LEVEL tracker */ msg_level=msg_level-1 RETURN 0 /* verify_paths END */ /* subroutine to handle the command line parameter values */ cli_parm_parse: PROCEDURE expose debug_flag msg_level backup_type, source_folder target_folder crlf /* Parse multiple args optionally enclosed in double quotes */ PARSE ARG Args Args = STRIP( Args) /* setup the variables */ calling_code ='cli_parm_parse' msg_level =msg_level+1 msg_text ='Initializing...' /* put a proc ENTRY into DEBUG */ CALL debug debug_flag calling_code msg_level msg_text msg_text="CLI input params : " CALL debug debug_flag calling_code msg_level msg_text||Args /* Init vars */ ARG. = '' ARG.0 = 0 Rest = Args /* DEBUG data */ msg_text="CLI params length = " length(Args) CALL debug debug_flag calling_code msg_level msg_text /* Parse Rest */ DO WHILE Rest <> '' /* Advance arg counter */ NumArgs = ARG.0 + 1 msg_text=NumArgs': Rest = ['Rest']' call debug debug_flag calling_code msg_level msg_text /* Parse Next from Rest */ IF LEFT( Rest, 1) = '"' THEN PARSE VAR Rest '"'Next'"' Rest ELSE PARSE VAR Rest Next Rest msg_text=NumArgs': Next = ['Next']' CALL debug debug_flag calling_code msg_level msg_text /* Strip multiple spaces between args */ Rest = STRIP( Rest) /* Save Next to Arg. array */ ARG.NumArgs = Next ARG.0 = NumArgs END /* Test Arg. array */ DO i = 1 TO ARG.0 msg_text='Arg.'i' = 'ARG.i CALL debug debug_flag calling_code msg_level msg_text END /* assign the results to the MAIN program variables */ backup_type = ARG.1 source_folder = ARG.2 target_folder = ARG.3 /* put a proc EXIT into DEBUG */ msg_text='Exiting...' CALL debug debug_flag calling_code msg_level msg_text /* reset DEBUG LEVEL tracker */ msg_level=msg_level-1 RETURN 0 /* cli_parm_parse END */ /* subroutine to build the two arrays which store RSYNC parameters - this is where the user can make specific adjustments to what RSYNC options - should be utilized */ build_rsync_parameter_arrays: PROCEDURE expose debug_flag msg_level crlf, rsync_param_local., rsync_param_remote. /* setup the variables */ calling_code ='build_rsync_parameter_arrays' msg_level =msg_level+1 msg_text ='Initializing...' /* put a proc ENTRY into DEBUG */ CALL debug debug_flag calling_code msg_level msg_text /* here is where we define the actual RSYNC parameters */ /* - LOCAL options - depending on the LOCAL FS capability these may need to be adjusted to - support the handling of the OS/2 extended attributes */ /* store how many option parameter entries we have, adjust as needed */ rsync_param_local.0.0 = 15 rsync_param_local.1.1 ="--dry-run" rsync_param_local.1.2 ="perform a trial run with no changes made" rsync_param_local.1.3 ="OFF" rsync_param_local.2.1 ="--archive" rsync_param_local.2.2 ="archive mode; equals -rlptgoD (no -H,-A,-X)" rsync_param_local.2.3 ="ON" rsync_param_local.3.1 ="--delete" rsync_param_local.3.2 ="delete extraneous files from dest dirs" rsync_param_local.3.3 ="ON" rsync_param_local.4.1 ="--progress" rsync_param_local.4.2 ="show progress during transfer" rsync_param_local.4.3 ="ON" rsync_param_local.5.1 ="--stats" rsync_param_local.5.2 ="give some file-transfer stats" rsync_param_local.5.3 ="ON" rsync_param_local.6.1 ="--whole-file" rsync_param_local.6.2 ="copy files whole (w/o delta-xfer algorithm)" rsync_param_local.6.3 ="ON" rsync_param_local.7.1 ="--no-xattrs" rsync_param_local.7.2 ="do not preserve extended attributes" rsync_param_local.7.3 ="OFF" rsync_param_local.8.1 ="--xattrs" rsync_param_local.8.2 ="preserve extended attributes" rsync_param_local.8.3 ="ON" rsync_param_local.9.1 ="--os2-perms" rsync_param_local.9.2 ="preserve OS/2 permissions and attributes" rsync_param_local.9.3 ="ON" rsync_param_local.10.1 ="--compress" rsync_param_local.10.2 ="compress file data during the transfer" rsync_param_local.10.3 ="OFF" rsync_param_local.11.1 ="--itemize-changes" rsync_param_local.11.2 ="output a change-summary for all updates" rsync_param_local.11.3 ="OFF" rsync_param_local.12.1 ="--human-readable" rsync_param_local.12.2 ="output numbers in a human-readable format" rsync_param_local.12.3 ="ON" rsync_param_local.13.1 ="--quiet" rsync_param_local.13.2 ="suppress non-error messages" rsync_param_local.13.3 ="ON" rsync_param_local.14.1 ="--verbose" rsync_param_local.14.2 ="increase verbosity" rsync_param_local.14.3 ="OFF" rsync_param_local.15.1 ="--inplace" rsync_param_local.15.2 ="update destination files in-place" rsync_param_local.15.3 ="OFF" msg_text="Completed the LOCAL array build..." CALL debug debug_flag calling_code msg_level msg_text /* - REMOTE options: - depending on the REMOTE capability these may need to be adjusted to support - the handling of the OS/2 extended attributes */ /* store how many option parameter entries we have, adjust as needed */ rsync_param_remote.0.0 = 15 rsync_param_remote.1.1 ="--dry-run" rsync_param_remote.1.2 ="perform a trial run with no changes made" rsync_param_remote.1.3 ="OFF" rsync_param_remote.2.1 ="--archive" rsync_param_remote.2.2 ="archive mode; equals -rlptgoD (no -H,-A,-X)" rsync_param_remote.2.3 ="ON" rsync_param_remote.3.1 ="--delete" rsync_param_remote.3.2 ="delete extraneous files from dest dirs" rsync_param_remote.3.3 ="ON" rsync_param_remote.4.1 ="--progress" rsync_param_remote.4.2 ="show progress during transfer" rsync_param_remote.4.3 ="ON" rsync_param_remote.5.1 ="--stats" rsync_param_remote.5.2 ="give some file-transfer stats" rsync_param_remote.5.3 ="ON" rsync_param_remote.6.1 ="--whole-file" rsync_param_remote.6.2 ="copy files whole (w/o delta-xfer algorithm)" rsync_param_remote.6.3 ="ON" rsync_param_remote.7.1 ="--no-xattrs" rsync_param_remote.7.2 ="do not preserve extended attributes" rsync_param_remote.7.3 ="ON" rsync_param_remote.8.1 ="--xattrs" rsync_param_remote.8.2 ="preserve extended attributes" rsync_param_remote.8.3 ="OFF" rsync_param_remote.9.1 ="--os2-perms" rsync_param_remote.9.2 ="preserve OS/2 permissions and attributes" rsync_param_remote.9.3 ="OFF" rsync_param_remote.10.1 ="--compress" rsync_param_remote.10.2 ="compress file data during the transfer" rsync_param_remote.10.3 ="OFF" rsync_param_remote.11.1 ="--itemize-changes" rsync_param_remote.11.2 ="output a change-summary for all updates" rsync_param_remote.11.3 ="OFF" rsync_param_remote.12.1 ="--human-readable" rsync_param_remote.12.2 ="output numbers in a human-readable format" rsync_param_remote.12.3 ="ON" rsync_param_remote.13.1 ="--quiet" rsync_param_remote.13.2 ="suppress non-error messages" rsync_param_remote.13.3 ="ON" rsync_param_remote.14.1 ="--verbose" rsync_param_remote.14.2 ="increase verbosity" rsync_param_remote.14.3 ="OFF" rsync_param_remote.15.1 ="--inplace" rsync_param_remote.15.2 ="update destination files in-place" rsync_param_remote.15.3 ="OFF" msg_text="Completed the REMOTE array build..." CALL debug debug_flag calling_code msg_level msg_text /* put a proc EXIT into DEBUG */ msg_text='Exiting...' CALL debug debug_flag calling_code msg_level msg_text /* reset DEBUG LEVEL tracker */ msg_level=msg_level-1 RETURN 0 /* build_rsync_parameter_arrays END */ /* subroutine to build the RSYNC command line parameters */ build_rsync_options: PROCEDURE expose debug_flag msg_level backup_type crlf, rsync_cli rsync_param_local., rsync_param_remote. /* setup the variables */ calling_code ='build_rsync_options' msg_level =msg_level+1 msg_text ='Initializing...' i =1 /* put a proc ENTRY into DEBUG */ CALL debug debug_flag calling_code msg_level msg_text /* how many entries to process ? */ msg_text="Array TYPE = "||backup_type||", "||, "Array SIZE = "||rsync_param_local.0.0 CALL debug debug_flag calling_code msg_level msg_text /* check the backup_type and go from there, either: LOCAL or REMOTE */ IF backup_type = 'LOCAL' THEN /* step through LOCAL the array and build up the RSYNC CLI */ DO WHILE i <= rsync_param_local.0.0 msg_text="Array processing loop (counter) = "||i CALL debug debug_flag calling_code msg_level msg_text msg_text="Control Flag set to >"||rsync_param_local.i.3||"<" CALL debug debug_flag calling_code msg_level msg_text msg_text=crlf||"parameter found : "||rsync_param_local.i.1||crlf||, "parameter info : "||rsync_param_local.i.2||crlf CALL debug debug_flag calling_code msg_level msg_text /* if the parameter is turned ON add to the list */ IF rsync_param_local.i.3 = "ON" THEN rsync_cli = rsync_cli||rsync_param_local.i.1||' ' /* increment the LOOP counter */ i=i+1 END ELSE /* step through the REMOTE array and build up the RSYNC CLI */ DO WHILE i <= rsync_param_local.0.0 msg_text="array processing loop (counter) = "||i CALL debug debug_flag calling_code msg_level msg_text msg_text="Control Flag set to >"||rsync_param_remote.i.3||"<" CALL debug debug_flag calling_code msg_level msg_text msg_text=crlf||"parameter found : "||rsync_param_remote.i.1||crlf||, "parameter info : "||rsync_param_remote.i.2||crlf CALL debug debug_flag calling_code msg_level msg_text /* if the parameter is turned ON add to the list */ IF rsync_param_remote.i.3 = "ON" THEN rsync_cli = rsync_cli||rsync_param_remote.i.1||' ' /* increment the LOOP counter */ i=i+1 END /* let's see what we put together */ msg_text = 'RSYNC CLI : '||rsync_cli CALL debug debug_flag calling_code msg_level msg_text /* put a proc EXIT into DEBUG */ msg_text='Exiting...' CALL debug debug_flag calling_code msg_level msg_text /* reset DEBUG LEVEL tracker */ msg_level=msg_level-1 RETURN 0 /* build_rsync_options END */ /* subroutine to encapsulate the string with single quotes */ Enquote: rc = 0 PARSE ARG Filename IF VERIFY( Filename, ' &|<>', 'M') <> 0 THEN /* Filename = "'"Filename"'" */ Filename = '"'Filename'"' RETURN( Filename) /* Enquote END */ /* subroutine to show DEBUG messages */ debug: PROCEDURE PARSE ARG debug_flag calling_code msg_level msg_text /* get my own msg_level variable for indentation loop */ msg_level_cntr = msg_level /* check if DEBUG=ON */ IF (debug_flag=1) THEN DO /* reset variables */ msg_indent='DEBUG : ' /* for each msg_level unit indent accordingly */ DO WHILE msg_level_cntr <> 0 msg_indent = msg_indent||' ==> ' msg_level_cntr = msg_level_cntr-1 END SAY msg_indent||calling_code||' => '||msg_text END RETURN 0 /* debug END */ /* subroutine to show LOG messages */ log: PROCEDURE PARSE ARG debug_flag calling_code msg_level msg_text /* check to see if DEBUG=ON, if so , run through DEBUG routine */ IF (debug_flag) THEN CALL debug debug_flag calling_code msg_level msg_text ELSE SAY msg_text RETURN 0 /* log END */