Speed enhancement. Changed the code to send products in batches.

2024-02-24 17:07:48 +01:00
3 changed files with 63 additions and 64 deletions
--- a/abacus2woo.py
+++ b/abacus2woo.py
@ -3,9 +3,10 @@ import sqlite3
 import sys
 import time
 import pyodbc
+import json
 from woocommerce import API

-# import all_from_woo
+import all_from_woo

 conn = sqlite3.connect('imd.db')  # intermediary db
 c = conn.cursor()
@ -17,15 +18,17 @@ c.execute('''CREATE TABLE IF NOT EXISTS products (id INTEGER NOT NULL PRIMARY KE

 # Connect to WooCommerce API
 wcapi = API(
-    url="https://www.rasterdoo.com/",
+    url="https://www.rasterdoo.com",
    consumer_key="ck_b3169b1723f6f39a965ed04ecfa77860fb89bbf5",
    consumer_secret="cs_a83f0217ed8d6191ab7de9df06a0c2b652f6bd57",
    wp_api=True,
    timeout=360,
-    version="wc/v2"
+    version="wc/v3"
 )
-print("Resting for for a bit...")
-# time.sleep(50)
+# Let's check if connection with WooCommerce is OK - list a sales report
+# print(wcapi.get("reports/sales?date_min=2024-01-01&date_max=2024-02-04").json())
+# print("Resting for for a bit...")
+# time.sleep(5)

 print("Connecting to MS SQL server")
 '''
@ -81,34 +84,34 @@ print("Abacus row in works:"), row  #
 snc = 0
 for row in cursor:
    while row is not None:
-        # print("Abacus row entered for loop and it is not None")
+        # print("Abacus row entered for loop & it is not None")
        # input("Press Enter to continue...") #
        aid = row[0]
        aprice = float(row[1])
        aname = row[2]
-        asifra = row[3]
+        asifra = str(row[3])
        aqty = int(row[4])
        agroup = row[5]
-        print("MS Sql group value:", agroup)
+        # print("MS Sql group value:", agroup)
        # This dictionary matches abacus product category id to equivalent id in woocommerce
        groupdict = {
            1: 21472, 2: 21473, 4: 21474, 5: 21475, 10: 21476, 11: 21477, 13: 21477, 15: 21477, 19: 21478, 24: 21479,
            30: 21480, 32: 21481, 33: 21482, 37: 21483, 38: 21484, 44: 21485, 45: 21486, 47: 21477, 48: 21477
        }
-        print("Coverted to Woocommerce group it becomes wgroup:", groupdict[agroup])
+        # print("Coverted to Woocommerce group it becomes wgroup:", groupdict[agroup])
        if agroup in groupdict:
            wgroup = groupdict[agroup]
-            print("Group in abacusu is from dictionary:", agroup, " - while woocommerce now holds:", wgroup)  #
+            # print("Group in abacusu is from dictionary:", agroup, " - while woocommerce now holds:", wgroup)  #
        else:
            wgroup = 3557
-            print("Group ", agroup, "does not exist in dictionary - we put it in woocommerce group -other-:", wgroup)  #
+            # print("Group ", agroup, "does not exist in dictionary - we put it in woocommerce group -other-:", wgroup)  #

        # Test id Woo product exists in SQLite table
        q = (asifra,)
        c.execute('SELECT * FROM products WHERE sifra=?', q)
-        print("Abacus data to be saved in SQLite:", aid, aprice, aname, asifra, aqty, wgroup)
+        # print("Abacus data to be saved in SQLite:", aid, aprice, aname, asifra, aqty, wgroup)
        # input("Press Enter to continue...") #
-        # We test SqlLite rows so that we update the table only with products that changed
+        # We test SqlLite rows so that we update the table only with product that changed
        trow = c.fetchone()
        if trow is not None:
            tnaziv = trow[3]
@ -118,30 +121,31 @@ for row in cursor:
            # print("There is an SQLite trow we need to check for an update: "), tnaziv, tprice, tqty, tgroup
            if aprice != tprice:
                snc = 1
-                print("bad price")
-            if aqty != tqty:
+                print(aname, "has a bad price")
+            elif aqty != tqty:
                snc = 1
-                print("bad qty")
-            if aname != tnaziv:
+                print(aname, "has a bad qty")
+            elif aname != tnaziv:
                snc = 1
-                print("bad naziv")
-            if wgroup != tgroup:
+                print(aname, "has bad naziv: ", tnaziv)
+            elif wgroup != tgroup:
                snc = 1
-                print(wgroup, "bad tgroup: ", tgroup)
+                print(wgroup, "has a bad tgroup: ", tgroup)
+            else:
+                snc = 0
            if snc == 1:
-                # print("=======")
+                print(asifra, aname, snc)
                c.execute('''UPDATE products SET aid = ?, price = ?, qty =?, naziv = ?, 
                            snc = ?, grupa = ? WHERE sifra = ?''',
                          (aid, aprice, aqty, aname, snc, wgroup, asifra))
                conn.commit()
                c.execute('SELECT * FROM products WHERE sifra=?', q)
                trow = c.fetchone()
-                print(trow)
-                print("=======")
+                # print(trow)
+                # print("=======")
                # input("Press Enter to continue...")
        else:
-            print("If sqlite trow does not exist (is ", trow, "), then first create one in Woo", aid, asifra, aname,
-                  aprice, aqty, wgroup)
+            print("If sqlite trow does not exist (is ", trow, "), then first create one in Woo", aid, asifra, aname, aprice, aqty, wgroup)
            # input("Press Enter to continue...") #
            aprice = str(aprice)
            data = {
@ -160,8 +164,6 @@ for row in cursor:
                ],
            }
            w = wcapi.post("products", data).json()
-
-
            # w = wcapi.get("products/categories").json()

            class ListStream:
@ -189,7 +191,7 @@ for row in cursor:
                          (rogueone, aid, asifra, aname, aprice, aqty, 1, wgroup))  # if no product it will create
                print("Inserting rogueone in SQLlite data:", rogueone, aid, asifra, aname, aprice, aqty, 1, wgroup)
                conn.commit()
-                # rgpath = "\"products/" + str(rogueone) + "\""
+                # rgpath = "\"product_batch/" + str(rogueone) + "\""
                # print rgpath
                # rg = wcapi.get(rogueone).json()
                # print("The rogueone:", rg)
@ -211,49 +213,45 @@ for row in cursor:
        # print("Resting for for a bit...")
        # time.sleep(5)
 conn.close()
+print("Wait before sending")
+# time.sleep(10)

 # Update Woo with our SQLite table
 conn = sqlite3.connect('imd.db')  # intermediary db
 c = conn.cursor()
 c.execute('''SELECT * FROM products WHERE snc = 1 ORDER BY naziv''')
-trow = c.fetchone()
-
-print("Selected for update:", trow)
-# cleantrow is not None:
 grupa = 0
+
+# Make a product_list
+product_list = list()
+trow = c.fetchone()
 while trow is not None:
-    id = trow[0]
+    # count += 1
+    # print("Selected for update:", trow)
+    tid = trow[0]
    tnaziv = trow[3]
    tprice = str(trow[4])
    tqty = trow[5]
    tgroup = trow[7]
-    # print type(id), type(tnaziv), type(tprice), type(tqty), "Grupa:", type(tgroup), tgroup
-    # input("Press enter ...")
-    data = {
-        "id": id,
-        "manage_stock": "true",
-        "stock_quantity": tqty,
-        "name": tnaziv,
-        "status": "publish",
-        "regular_price": tprice,
-        "categories": [
-            {
-                "id": tgroup
-            }
-        ],
-    }
-    # print data
-    uplink = "products/" + str(id)
-    # print uplink
-    up = wcapi.put((uplink), data).json()
-    print(up)
-    # input("Press enter ...")
-    if trow is None:
-        break
-    else:
-        trow = c.fetchone()
-    print("___________________________________________")
-    print("Selected next for update:", trow)
+    # product = [tid, tqty, tnaziv, tprice, tgroup]
+    product = {'id': tid, 'manage_stock': 'true', 'stock_quantity': tqty, 'name': tnaziv, 'status': 'publish', 'regular_price': tprice, 'categories': [{'id': tgroup}]}
+    # print(product)
+    product_list.append(product)
+    trow = c.fetchone()
+print(product_list)
+# Make batches of products and send them to WooCommerce
+l = len(product_list)
+print("Lenght: ", l)
+batch_size = 100
+
+for i in range(0, l, batch_size):
+    one_product_batch = product_list[i:i + batch_size]
+    data = {'update': one_product_batch}
+    # print(one_product_batch)
+    print("_________________________BATCH", i, "_________________________")
+    response = wcapi.post("products/batch", data).json()
+    print(response)
+
 c.execute('UPDATE products SET snc = 0')
 conn.commit()

--- a/all_from_woo.py
+++ b/all_from_woo.py
@ -19,7 +19,8 @@ def initSQLite():
        consumer_key="ck_b3169b1723f6f39a965ed04ecfa77860fb89bbf5",
        consumer_secret="cs_a83f0217ed8d6191ab7de9df06a0c2b652f6bd57",
        wp_api=True,
-        version="wc/v2"
+        timeout=360,
+        version="wc/v3"
    )
    r = wcapi.get("products")  # get woo web page where products are
    h = int(r.headers['X-WP-TotalPages'])  # in the header we see number of tot pages
@ -31,7 +32,7 @@ def initSQLite():
        r = wcapi.get(goto_page)
        page_txt = r.json()  # contents of one page ar stored in variable
        # We fill our SQLite table with Woo Products
-        for product in page_txt:  # first we go through products in this page
+        for product in page_txt:  # first we go through product in this page
            wsku = product.get('sku')
            wid = product.get('id')
            wname = product.get('name')
@ -41,9 +42,9 @@ def initSQLite():
                wqty = 0
            # FIX THIS! - It does not update - just inserts if new ...
            c.execute('INSERT INTO products VALUES (?,?,?,?,?,?,?,?)',
-                      (wid, "", wsku, wname, wprice, wqty, 0))  # if no product it will create
+                      (wid, "", wsku, wname, wprice, wqty, 0, ""))  # if no product it will create
            c.execute('UPDATE products SET aid = ?, sifra = ?, naziv = ?, price = ?, qty = ?, snc = ? WHERE id = ?',
-                      ("", wsku, wname, wprice, wqty, 0, wid, ""))
+                      ("", wsku, wname, wprice, wqty, 0, wid))
            print("Insert woocommerce data to SQLlite:", wid, "", wsku, wname, wprice, wqty, 0)
            conn.commit()

--- a/imd.db
+++ b/imd.db