pages/eng_13982_workday_the_system_detects_new_folders_in_sftp_as_files_for_processing.org

ENG-13982 - Workday: The system detects new folders in SFTP as files for processing

#+begin_src python :results output

import os

TEST_FOLDER_PREFIX = 'test'

def should_process_s3_key(s3_key: str) -> bool:

file_path, extension = os.path.splitext(s3_key)

# Keys look like: sftp/<company_id>/some/other/path/file.csv

is_csv_extension = extension.lower() == '.csv'

is_test_folder = False

split_path = file_path.split('/')

# If our split path is less than 3 in length, we have a root file: 'sftp/test.csv'

if len(split_path) > 3:

sub_folder_name = split_path[2].lower() # e.g. `some` in path above

# Folder starts with prefix and is not a CSV file

is_test_folder = sub_folder_name.startswith(TEST_FOLDER_PREFIX)

if is_test_folder:

print(f'Skipping "{s3_key}" key with sub-folder prefixed with `test`')

return False

if not is_csv_extension:

print(f'Skipping "{s3_key}" key because it does not have a CSV file extension')

return False

return True

print('sftp/23432/test.csv - ', should_process_s3_key('sftp/23432/test.csv'), ' == True')

print('sftp/23432/testFile.csv - ', should_process_s3_key('sftp/23432/testFile.csv'), ' == True')

print('sftp/23432/test/File.csv - ', should_process_s3_key('sftp/23432/test/File.csv'), ' == False')

print('sftp/23432/test/File.txt - ', should_process_s3_key('sftp/23432/test/File.txt'), ' == False')

print('sftp/23432/File.txt - ', should_process_s3_key('sftp/23432/File.txt'), ' == False')

print('sftp/23432/File.csv - ', should_process_s3_key('sftp/23432/File.csv'), ' == True')

#+end_src

#+RESULTS:

: sftp/23432/test.csv - True == True

: sftp/23432/testFile.csv - True == True

: Skipping "sftp/23432/test/File.csv" key with sub-folder prefixed with `test`

: sftp/23432/test/File.csv - False == False

: Skipping "sftp/23432/test/File.txt" key with sub-folder prefixed with `test`

: sftp/23432/test/File.txt - False == False

: Skipping "sftp/23432/File.txt" key because it does not have a CSV file extension

: sftp/23432/File.txt - False == False

: sftp/23432/File.csv - True == True