100 lines
3.3 KiB
Python
100 lines
3.3 KiB
Python
import mariadb
|
|
import json
|
|
from dotenv import load_dotenv
|
|
import os
|
|
|
|
# Load environment variables from .env file
|
|
load_dotenv()
|
|
|
|
def connect_to_db():
|
|
try:
|
|
# Read the needed variables from the environment
|
|
conn = mariadb.connect(
|
|
user=os.getenv("DB_USER"),
|
|
password=os.getenv("DB_PASSWORD"),
|
|
host=os.getenv("DB_HOST"),
|
|
port=int(os.getenv("DB_PORT")),
|
|
database=os.getenv("DB_NAME")
|
|
)
|
|
return conn
|
|
except mariadb.Error as e:
|
|
print(f"Error connecting to MariaDB: {e}")
|
|
return None
|
|
|
|
def insert_entity(cursor, cik, entity_name):
|
|
cursor.execute(
|
|
"INSERT IGNORE INTO entities (cik, name) VALUES (?, ?)", (cik, entity_name))
|
|
|
|
def insert_fact(cursor, taxonomy, fact_id, label, description, unit):
|
|
cursor.execute(
|
|
"INSERT IGNORE INTO facts (id, taxonomy, label, description, unit) VALUES (?, ?, ?, ?, ?)",
|
|
(fact_id, taxonomy, label, description, unit)
|
|
)
|
|
|
|
def insert_data(cursor, cik, fact_id, start, end, val, accn, fy, fp, form, filed, frame):
|
|
cursor.execute(
|
|
"""INSERT IGNORE INTO data (cik, fact_id, start, end, val, accn, fy, fp, form, filed, frame)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
(cik, fact_id, start, end, val, accn, fy, fp, form, filed, frame)
|
|
)
|
|
|
|
def parse_json_and_insert_data(file_path, cursor):
|
|
with open(file_path, 'r') as file:
|
|
data = json.load(file)
|
|
|
|
# Insert the entity
|
|
cik = data['cik']
|
|
entity_name = data['entityName']
|
|
insert_entity(cursor, cik, entity_name)
|
|
|
|
# Iterate over facts
|
|
for taxonomy, fact_details in data['facts'].items():
|
|
for fact_id, fact in fact_details.items():
|
|
# Get fact details
|
|
label = fact.get('label')
|
|
description = fact.get('description')
|
|
|
|
for unit, unit_vals in fact.get('units', {}).items():
|
|
# Insert fact (taxonomy level doesn't seem directly stored in JSON)
|
|
insert_fact(cursor, taxonomy, fact_id, label, description, unit)
|
|
|
|
# Insert each data point
|
|
for entry in unit_vals:
|
|
start = entry.get('start', None)
|
|
end = entry['end']
|
|
val = entry['val']
|
|
accn = entry['accn']
|
|
fy = entry['fy']
|
|
fp = entry['fp']
|
|
form = entry['form']
|
|
filed = entry['filed']
|
|
frame = entry.get('frame', None)
|
|
|
|
insert_data(cursor, cik, fact_id, start, end, val, accn, fy, fp, form, filed, frame)
|
|
|
|
def main():
|
|
# Connect to the database
|
|
conn = connect_to_db()
|
|
if conn is None:
|
|
return
|
|
|
|
try:
|
|
cursor = conn.cursor()
|
|
|
|
# Load JSON and insert data
|
|
json_file_path = 'CIK0000320193.json'
|
|
parse_json_and_insert_data(json_file_path, cursor)
|
|
|
|
# Commit the transaction
|
|
conn.commit()
|
|
|
|
except Exception as e:
|
|
print(f"Error occurred: {e}")
|
|
conn.rollback()
|
|
finally:
|
|
cursor.close()
|
|
conn.close()
|
|
print("Connection closed")
|
|
|
|
if __name__ == "__main__":
|
|
main() |