Python: upload multiple files concurrently with aiohttp and show progress bars with tqdm

ndrbrt

Andrea Bertoloni

Posted on November 15, 2021

Python: upload multiple files concurrently with aiohttp and show progress bars with tqdm

I had to quickly implement this functionality in a command line tool and tried to look for some online ready-to-use example.
Since I wasn't able to find any, I came up with this solution (perhaps someone can find it helpful).

import os
import asyncio
import aiohttp
import aiofiles
from tqdm import tqdm


class FileManager():
    def __init__(self, file_name: str):
        self.name = file_name
        self.size = os.path.getsize(self.name)
        self.pbar = None

    def __init_pbar(self):
        self.pbar = tqdm(
            total=self.size,
            desc=self.name,
            unit='B',
            unit_scale=True,
            unit_divisor=1024,
            leave=True)

    async def file_reader(self):
        self.__init_pbar()
        chunk_size = 64*1024
        async with aiofiles.open(self.name, 'rb') as f:
            chunk = await f.read(chunk_size)
            while chunk:
                self.pbar.update(chunk_size)
                yield chunk
                chunk = await f.read(chunk_size)
            self.pbar.close()


async def upload(file: FileManager, url: str, session: aiohttp.ClientSession):
    try:
        async with session.post(url, data=file.file_reader()) as res:
            # NB: if you also need the response content, you have to await it
            return res
    except Exception as e:
        # handle error(s) according to your needs
        print(e)


async def main(files):
    url = 'https://httpbin.org/post'
    files = [FileManager(file) for file in files]

    async with aiohttp.ClientSession() as session:
        res = await asyncio.gather(*[upload(file, url, session) for file in files])

    print(f'All files have been uploaded ({len(res)})')


if __name__ == "__main__":
    # In a real application you may may want to handle files
    # in a more robust way, do some validation etc.
    files = ['./file_1', './file_2', './file_3']

    asyncio.run(main(files))

Enter fullscreen mode Exit fullscreen mode
💖 💪 🙅 🚩
ndrbrt
Andrea Bertoloni

Posted on November 15, 2021

Join Our Newsletter. No Spam, Only the good stuff.

Sign up to receive the latest update from our blog.

Related