You can use aws S3
You can run one python script on your instance which uploads the json file to s3 whenever the json gets generated and another python script on local machine where you can use (script for sqs queue and s3 download configuration) or (script which downloads the latest file uploaded to s3 bucket).
Case1:
Whenever the json file gets uploaded to S3 you will get message in the sqs queue that the file has been uploaded to s3 and then the file gets downloaded to your local machine.
Case2:
Whenever the json file gets uploaded to s3, you can run the download script which downloads the latest json file.
upload.py:
import boto3
import os
import socket
def upload_files(path):
session = boto3.Session(
aws_access_key_id='your access key id',
aws_secret_access_key='your secret key id',
region_name='region'
)
s3 = session.resource('s3')
bucket = s3.Bucket('bucket name')
for subdir, dirs, files in os.walk(path):
for file in files:
full_path = os.path.join(subdir, file)
print(full_path[len(path)+0:])
with open(full_path, 'rb') as data:
bucket.put_object(Key=full_path[len(path)+0:], Body=data)
if __name__ == "__main__":
upload_files('your pathwhich in your case is (/home/ubuntu/)')
your other script on local machine:
download1.py with sqs queue
import boto3
import logzero
from logzero import logger
s3_resource = boto3.resource('s3')
sqs_client=boto3.client('sqs')
### Queue URL
queue_url = 'queue url'
### aws s3 bucket
bucketName = "your bucket-name"
### Receive the message from SQS queue
response_message = sqs_client.receive_message(
QueueUrl=queue_url,
MaxNumberOfMessages=1,
MessageAttributeNames=[
'All'
],
)
message=response_message['Messages'][0]
receipt_handle = message['ReceiptHandle']
messageid=message['MessageId']
filename=message['Body']
try:
s3_resource.Bucket(bucketName).download_file(filename,filename)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code']=='404':
logger.info("The object does not exist.")
else:
raise
logger.info("File Downloaded")
download2.py with latest file downloading from s3:
import boto3
### S3 connection
s3_resource = boto3.resource('s3')
s3_client = boto3.client('s3')
bucketName = 'your bucket-name'
response = s3_client.list_objects_v2(Bucket=bucketName)
all = response['Contents']
latest = max(all, key=lambda x: x['LastModified'])
s3 = boto3.resource('s3')
key=latest['Key']
print("downloading file")
s3_resource.Bucket(bucketName).download_file(key,key)
print("file download")
