Advertisement
VladNitu

save_fast_postgresql_mogrify_vlad

Jun 17th, 2023
132
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 2.37 KB | None | 0 0
  1.     def save(self):
  2.         """
  3.        This method saves a NewsDocument in the database.
  4.        :return: nothing / throws an error
  5.        """
  6.         # Create a cursor that returns a dictionary as a result
  7.         cur = conn.cursor()
  8.  
  9.         try:
  10.             # Insert the url into the urls table and retrieve its id
  11.             cur.execute(
  12.                 f"""
  13.                INSERT INTO {schema}.urls (url) VALUES (%s)
  14.                ON CONFLICT (url) DO NOTHING
  15.                RETURNING id
  16.                """, (self.url,))
  17.  
  18.             # Fetch the result
  19.             doc = cur.fetchone()
  20.  
  21.             if doc:
  22.                 url_id = doc[0]
  23.                 print(url_id)
  24.  
  25.                 # These are the fingerprints that have to be inserted
  26.                 new_fingerprints = [fp for fp in self.fingerprints if fp not in existing_fps]
  27.  
  28.                 # If there are new fingerprints, insert them into the fingerprints table
  29.                 if new_fingerprints:
  30.                     values_sql = ', '.join(cur.mogrify('(%s)', (fp,)).decode() for fp in new_fingerprints)
  31.                     cur.execute(
  32.                         f"""
  33.                        INSERT INTO {schema}.fingerprints (fingerprint)
  34.                        VALUES {values_sql}
  35.                        ON CONFLICT (fingerprint) DO NOTHING
  36.                        """)
  37.  
  38.                     # Update existing_fps with the new fingerprints
  39.                     existing_fps.update(new_fingerprints)
  40.  
  41.                 # Prepare the data for the url_fingerprints
  42.                 url_fingerprints_data = [(url_id, fp) for fp in self.fingerprints]
  43.  
  44.                 values_sql = ', '.join(cur.mogrify('(%s, %s)', row).decode() for row in url_fingerprints_data)
  45.                 cur.execute(
  46.                     f"""
  47.                    INSERT INTO {schema}.url_fingerprints (url_id, fingerprint_id)
  48.                    VALUES {values_sql}
  49.                    ON CONFLICT DO NOTHING
  50.                    """)
  51.  
  52.             # Commit all changes at once
  53.             conn.commit()
  54.  
  55.         # In case of a database error, we roll back any commits that have been made
  56.         except psycopg2.Error as e:
  57.             print(f"Could not insert data: {e}")
  58.             # Rollback the current transaction if there's any error
  59.             conn.rollback()
  60.  
  61.         # Close the cursor
  62.         cur.close()
  63.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement