Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def save(self):
- """
- This method saves a NewsDocument in the database.
- :return: nothing / throws an error
- """
- # Create a cursor that returns a dictionary as a result
- cur = conn.cursor()
- try:
- # Insert the url into the urls table and retrieve its id
- cur.execute(
- f"""
- INSERT INTO {schema}.urls (url) VALUES (%s)
- ON CONFLICT (url) DO NOTHING
- RETURNING id
- """, (self.url,))
- # Fetch the result
- doc = cur.fetchone()
- if doc:
- url_id = doc[0]
- print(url_id)
- # These are the fingerprints that have to be inserted
- new_fingerprints = [fp for fp in self.fingerprints if fp not in existing_fps]
- # If there are new fingerprints, insert them into the fingerprints table
- if new_fingerprints:
- values_sql = ', '.join(cur.mogrify('(%s)', (fp,)).decode() for fp in new_fingerprints)
- cur.execute(
- f"""
- INSERT INTO {schema}.fingerprints (fingerprint)
- VALUES {values_sql}
- ON CONFLICT (fingerprint) DO NOTHING
- """)
- # Update existing_fps with the new fingerprints
- existing_fps.update(new_fingerprints)
- # Prepare the data for the url_fingerprints
- url_fingerprints_data = [(url_id, fp) for fp in self.fingerprints]
- values_sql = ', '.join(cur.mogrify('(%s, %s)', row).decode() for row in url_fingerprints_data)
- cur.execute(
- f"""
- INSERT INTO {schema}.url_fingerprints (url_id, fingerprint_id)
- VALUES {values_sql}
- ON CONFLICT DO NOTHING
- """)
- # Commit all changes at once
- conn.commit()
- # In case of a database error, we roll back any commits that have been made
- except psycopg2.Error as e:
- print(f"Could not insert data: {e}")
- # Rollback the current transaction if there's any error
- conn.rollback()
- # Close the cursor
- cur.close()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement