Org Web Adapter

Editing pages/eng_13982_workday_the_system_detects_new_folders_in_sftp_as_files_for_processing.org

:PROPERTIES:
:ID:       0e2ab07e-3d9a-419c-bf23-884bfd403a67
:END:
#+title: ENG-13982 - Workday: The system detects new folders in SFTP as files for processing

- tags :: [[id:3fe002d7-c62b-4efe-b67d-362bc35c7888][Devops]] [[id:b08ac197-337b-4392-84d5-98eaead61400][15Five]] [[id:b71b60df-d439-4aa1-8c2c-265707adf8ec][Jira]] [[id:dd83d825-692f-4b26-9ceb-4569e9d9ec4b][Terraform]]  [[id:48ec7ad9-9d6e-47ec-878e-6d1dce1a830a][Workday Integration with 15Five]]
- source :: https://15five-dev.atlassian.net/browse/ENG-139892
- pr :: https://github.com/15five/fifteen5-terraform-modules/pull/234

#+begin_src python :results output
import os
TEST_FOLDER_PREFIX = 'test'
def should_process_s3_key(s3_key: str) -> bool:
    file_path, extension = os.path.splitext(s3_key)
    # Keys look like: sftp/<company_id>/some/other/path/file.csv
    is_csv_extension = extension.lower() == '.csv'
    is_test_folder = False

split_path = file_path.split('/')
    # If our split path is less than 3 in length, we have a root file: 'sftp/test.csv'
    if len(split_path) > 3:
        sub_folder_name = split_path[2].lower()  #  e.g. `some` in path above
        # Folder starts with prefix and is not a CSV file
        is_test_folder = sub_folder_name.startswith(TEST_FOLDER_PREFIX)

if is_test_folder:
        print(f'Skipping "{s3_key}" key with sub-folder prefixed with `test`')
        return False

if not is_csv_extension:
        print(f'Skipping "{s3_key}" key because it does not have a CSV file extension')
        return False

return True
print('sftp/23432/test.csv - ', should_process_s3_key('sftp/23432/test.csv'), ' == True')
print('sftp/23432/testFile.csv - ', should_process_s3_key('sftp/23432/testFile.csv'), ' == True')
print('sftp/23432/test/File.csv - ', should_process_s3_key('sftp/23432/test/File.csv'), ' == False')
print('sftp/23432/test/File.txt - ', should_process_s3_key('sftp/23432/test/File.txt'), ' == False')
print('sftp/23432/File.txt - ', should_process_s3_key('sftp/23432/File.txt'), ' == False')
print('sftp/23432/File.csv - ', should_process_s3_key('sftp/23432/File.csv'), ' == True')
  #+end_src

#+RESULTS:
  : sftp/23432/test.csv -  True  == True
  : sftp/23432/testFile.csv -  True  == True
  : Skipping "sftp/23432/test/File.csv" key with sub-folder prefixed with `test`
  : sftp/23432/test/File.csv -  False  == False
  : Skipping "sftp/23432/test/File.txt" key with sub-folder prefixed with `test`
  : sftp/23432/test/File.txt -  False  == False
  : Skipping "sftp/23432/File.txt" key because it does not have a CSV file extension
  : sftp/23432/File.txt -  False  == False
  : sftp/23432/File.csv -  True  == True