Argthtjtr

I have a software that runs continuously and that periodically read from db. On some platform we observed that sometimes the reads were very slow and we figured out that it was due to the cache cleaning done by the operative system.

I have replicated the issue in the following script:

import subprocess

from subprocess import call

import time

import pandas as pd

import numpy as np

from sqlalchemy.orm import sessionmaker

from sqlalchemy import func, distinct, text

from sqlalchemy.ext.hybrid import hybrid_method

from sqlalchemy.ext.declarative import declarative_base

from sqlalchemy import Column, Integer, String, create_engine, and_

import os







n_users = 1000

n_days = 60

n_domains = 100

all_users = ['user%d' % i for i in range(n_users)]

all_domains = ['domain%d' % i for i in range(n_domains)]

n_rows = n_users*n_days*n_domains





Base = declarative_base()



#file_path = '/home/local/CORVIL/lpuggini/Desktop/example.db'

file_path = '/data/misc/luca/example.db'

db_path = 'sqlite:///' + file_path





engine = create_engine(db_path)





def get_session():

    Session = sessionmaker(bind=engine)

    session = Session()

    Base.metadata.create_all(engine)

    return session





class DailyUserWebsite(Base):

    __tablename__ = 'daily_user_website'



    id = Column(Integer, primary_key=True)

    user = Column(String(600), index=True)

    domain = Column(String(600))

    time_secs = Column(Integer, index=True)



    def __repr__(self):

        return "DailyUserWebsite(user='%s', domain='%s', time_secs=%d)" % 

            (self.user, self.domain, self.time_secs)





def get_df_daily_data_per_users(users):

    session = get_session()

    query = session.query(DailyUserWebsite).filter(DailyUserWebsite.user.in_(users))

    df = pd.read_sql(query.statement, query.session.bind)

    session.close()

    return df





def create_db():

    if os.path.exists(file_path):

        os.remove(file_path)



    session = get_session()

    batch_size = 10000

    n_iter = int(n_rows / batch_size) + 1

    for i in range(n_iter):

        print 'Building db iteration %d out of %d' % (i, n_iter)

        df = pd.DataFrame()

        df['user'] = np.random.choice(all_users, batch_size)

        df['domain'] = np.random.choice(all_domains, batch_size)

        df['time_secs'] = [x - x%(3600*24) for x in np.random.randint(0, 3600*24*60, batch_size)]

        df.to_sql('daily_user_website', engine, if_exists='append', index=False)





create_db()

for i in range(20):

    users = np.random.choice(all_users, 200)

    t0 = time.time()

    df = get_df_daily_data_per_users(users)

    t1 = time.time()

    print 'it=', i, 'time taken to read %d rows %f ' % (df.shape[0],  t1-t0)

    if i % 5 == 0:

        print 'Clean cache'

        os.system("sync; echo 3 > /proc/sys/vm/drop_caches")

That generates the following outputs:

(samenv) probe686:/data/misc/luca # python db_test.py

it= 0 time taken to read 1089089 rows 8.058407 

Clean cache

it= 1 time taken to read 1099234 rows 104.352085 

it= 2 time taken to read 1087292 rows 8.189860 

it= 3 time taken to read 1077284 rows 8.176948 

it= 4 time taken to read 1057111 rows 7.980002 

it= 5 time taken to read 1075694 rows 8.144479 

Clean cache

it= 6 time taken to read 1117925 rows 106.357740 

it= 7 time taken to read 1124208 rows 8.523779 

it= 8 time taken to read 1083049 rows 8.368766 

it= 9 time taken to read 1112264 rows 9.233548 

it= 10 time taken to read 1098628 rows 8.316519 

Clean cache

Is there any way to improve speed after a cache cleaning or to mitigate the effect?