Why my code still slow after threading for 15k records only, how to fix this

Question

I have a script, taking links from a file, visiting it, getting re-directed links, storing it back. But it works too slow on a file with 15k records. How can I make it quick? already used threading Please do help to fix it out!, I've tried multiple ways, threadings but I cannot make it quick. Is there any solution to my problem by any chance? any expert who could help me out.

import concurrent.futures
import sys
import pandas as pd
import requests
from threading import Thread
from queue import Queue
out_put_file=""
linkes = None
out = []
urls = []
old = []
file_name =None
concurrent = 10000
q = None
count=0
df =None
def do_work():
    while True:
        global q
        url = q.get()
        res = get_status(url)
        q.task_done()
def get_status(o_url):
    try:
        res = requests.get(o_url)
        if res:
            out.append(res.url)
            old.append(o_url)
        print(count)
        count=count+1
        return [res.status_code,res.url ,o_url]
    except:
        pass
    return [ans.status_code,ans.url,url]
def process_data():
        global q
        global file_name
        global linkes
        global df
        file_name = input("Enter file name : ")
        file_name = file_name.strip()
        print("Generating .......")
        df = pd.read_csv(file_name+".csv")
        old_links =df["shopify"]
        for i in old_links:
            if type(i)!=str:
                urls.append(i)
                continue
            if not i.startswith("http"):
                
                linkes = "http://"+i 
                urls.append(linkes)
            else:
                urls.append(i)
        df["shopify"]=urls
        q = Queue(concurrent * 2)
        for i in range(concurrent):
            t = Thread(target=do_work)
            t.daemon = True
            t.start()
        try:
            for url in urls:
                if type(url)!=str:
                    continue
                q.put(url.strip())
            q.join()
        except KeyboardInterrupt:
            sys.exit(1)
process_data()
for i in range (len(df['shopify'])):
    for j in range(len(old)):
        if df['shopify'][i]==old[j]:
            df['shopify'][i]=out[j]
df = df[~df['shopify'].astype(str).str.startswith('http:')]
df = df.dropna()
df.to_csv(file_name+"-new.csv",index=False)

Email,shopify,Proofy_Status_Name
[email protected],http://puravidabracelets.myshopify.com,Deliverable
[email protected],cafe-select.co.uk,Deliverable
[email protected],,Deliverable
[email protected],stevessnacks.com,Deliverable
[email protected],runwayriches.com,Deliverable
[email protected],blackdogride.com.au,Deliverable
[email protected],grass4you.com,Deliverable
[email protected],prideandprestigehair.com,Deliverable
[email protected],dancinwoofs.com,Deliverable

Answer 1

Threads in Python do not run simultaneously due to the Global Interpreter Lock. You might want to use the multiprocessing module instead, or ProcessPoolExecutor() from concurrent.futures. If you decide to use ProcessPoolExecutors, pass the URLs to the callback and have the callback return the old and redirected URL which should be returned by the result method of the future you get from the executor.submit. When using processes, global variables are not shared, unlike threads.

There has been an attempt to remove the global interpreter lock but without the GIL, Python doesn't run quite as fast or something like that if I remember correctly.

Something like the following might work. I renamed the concurrent variable because it would shadow the concurrent module and probably cause an error. This code is untested because I don't have the csv file to test with.

import concurrent.futures
from concurrent.futures import ProcessPoolExecutor
import sys
import pandas as pd
import requests
import numpy as np
from threading import Thread
from queue import Queue
out_put_file=""
linkes = None
out = []
urls = []
old = []
futures = []
file_name =None
concurrent_ = 10000
q = None
count=0
df =None
def do_work(urls):
    results = []
    for url in urls:
        res = get_status(url)
        if res:
            results.append((res[2], res[1]))
        else:
            results.append((url, url))
    return results
def get_status(o_url):
    try:
        res = requests.get(o_url)
        if res:
            out.append(res.url)
            old.append(o_url)
        #print(count)
        #count=count+1
        return [res.status_code,res.url ,o_url]
    except:
        pass
def load_url(url, timeout):
    ans = requests.get(url, timeout=timeout)
    return [ans.status_code,ans.url,url]
def process_data():
        global q
        global file_name
        global linkes
        global df
        global urls
        file_name = input("Enter file name : ")
        file_name = file_name.strip()
        print("Generating .......")
        df = pd.read_csv(file_name+".csv")
        old_links =df["shopify"]
        for i in old_links:
            if type(i)!=str:
                urls.append(i)
                continue
            if not i.startswith("http"):
                
                linkes = "http://"+i 
                urls.append(linkes)
            else:
                urls.append(i)
        df["shopify"]=urls
        workers = 50
        with ProcessPoolExecutor(max_workers=workers) as executor:
            url_arrays = np.array_split(urls, workers)
            for urls in url_arrays:
                f = executor.submit(do_work, urls)
                futures.append(f)
process_data()
df['shopify'] = [res[1] for f in concurrent.futures.as_completed(futures) for res in f.result()]
df = df[~df['shopify'].astype(str).str.startswith('http:')]
df = df.dropna()
df.to_csv(file_name+"-new.csv",index=False)