4장. 데이터 수집: 데이터 추출

parser = configparser.ConfigParser() # IAM 유저 생성 후 발급되는 SECRET KEY, ACCESS KEY를 의미 parser.read("pipeline.conf") access_key = parser.get("aws_boto_credentials", "access_key") secret_key = parser.get("aws_boto_credentials", "secret_key") bucket_name = parser.get("aws_boto_credentials", "bucket_name") s3 = boto3.client('s3', aws_access_key_id = access_key, aws_secret_access_key = secret_key) s3_file = '''csv file name''' s3.upload_file('''csv file''', bucket_name, s3_file)

select variable_value as bin_log_status from performance_schema.global_variables where variable_name='log_bin'

# 세팅 후라고 가정 b_stream = BinLogStreamReader( connection_settings = mysql_settings, server_id=100, only_events=[ row_event.DeleteRowsEvent, row_event.WriteRowsEvent, row_event.UpdateRowsEvent ] ) for event in b_stream: event.dump() b_stream.close()

from pymongo import MongoClient import csv import boto3 import datetime from datetime import timedelta import configparser parser = configparser.ConfigParser() parser.read("pipeline.conf") hostname = parser.get("mongo_config", "hostname") username = parser.get("mongo_config", "username") password = parser.get("mongo_config", "password") database_name = parser.get("mongo_config", "database") collection_name = parser.get("mongo_config", "collection") mongo_client = MongoClient( "mongodb+srv://" + username + ":" + password + "@" + hostname + "/" + database_name + "?retryWrites=true&" + "w=majority&ssl=true&" + "ssl_cert_reqs=CERT_NONE") # connect to the db where the collection resides mongo_db = mongo_client[database_name] # choose the collection to query documents from mongo_collection = mongo_db[collection_name] start_date = datetime.datetime.today() + timedelta(days = -1) end_date = start_date + timedelta(days = 1 ) mongo_query = { "$and":[{"event_timestamp" : { "$gte": start_date }}, {"event_timestamp" : { "$lt": end_date }}] } event_docs = mongo_collection.find(mongo_query, batch_size=3000) # create a blank list to store the results all_events = [] # iterate through the cursor for doc in event_docs: # Include default values event_id = str(doc.get("event_id", -1)) event_timestamp = doc.get( "event_timestamp", None) event_name = doc.get("event_name", None) # add all the event properties into a list current_event = [] current_event.append(event_id) current_event.append(event_timestamp) current_event.append(event_name) # add the event to the final list of events all_events.append(current_event) export_file = "export_file.csv" with open(export_file, 'w') as fp: csvw = csv.writer(fp, delimiter='|') csvw.writerows(all_events) fp.close()

lat = 42.36 lon = 71.05 lat_log_params = {"lat": lat, "lon": lon} api_response = requests.get( "http://api.open-notify.org/iss-pass.json", params=lat_log_params) # create a json object from the response content response_json = json.loads(api_response.content) all_passes = [] for response in response_json['response']: current_pass = [] #store the lat/log from the request current_pass.append(lat) current_pass.append(lon) # store the duration and risetime of the pass current_pass.append(response['duration']) current_pass.append(response['risetime']) all_passes.append(current_pass) export_file = "export_file.csv" with open(export_file, 'w') as fp: csvw = csv.writer(fp, delimiter='|') csvw.writerows(all_passes) fp.close()