/* * StoreCache.cc * * Created on: Jan 6, 2011 * Author: lamikr */ #include #include #include "log.h" #include "Store.hh" #include "StoreCache.hh" #include "StoreDay.hh" #include "DeviceConfig.hh" #include "FileUtil.hh" using namespace std; using namespace plp; StoreCache::StoreCache(string device_id_param, Date *date_time_param): Store(device_id_param, date_time_param) { } StoreCache::~StoreCache() { } string StoreCache::get_dir_name(string device_id_param, Date *date_time_param, EnumSummaryPeriod period_type_param, EnumSummaryCalculationType calc_type_param) { string ret_val; char buffer[30]; string bd_name; string period_name; string calc_type; period_name = DataSummary::get_summary_period_name(period_type_param); calc_type = DataSummary::get_summary_calculation_name(calc_type_param); bd_name = DeviceConfig::get_base_dir_name(); bd_name = FileUtil::concat_paths(bd_name, CACHE_DIR_NAME); bd_name = FileUtil::concat_paths(bd_name, device_id_param); bd_name = FileUtil::concat_paths(bd_name, period_name); bd_name = FileUtil::concat_paths(bd_name, calc_type); if (period_type_param == PERIOD_YEARLY) { ret_val = bd_name; } else if (period_type_param == PERIOD_MONTHLY) { snprintf(buffer, 30, "%d", date_time_param->year); ret_val = bd_name + "/" + buffer; } else { snprintf(buffer, 30, "%d/%02d", date_time_param->year, date_time_param->month); ret_val = bd_name + "/" + buffer; } return ret_val; } string StoreCache::get_file_name(string device_id_param, Date *date_time_param, EnumSummaryPeriod period_type_param, EnumSummaryCalculationType calc_type_param) { string ret_val; string fname; char buffer[30]; if (period_type_param == PERIOD_YEARLY) { snprintf(buffer, 30, "%d", date_time_param->year); } else if (period_type_param == PERIOD_MONTHLY) { snprintf(buffer, 30, "%d-%02d", date_time_param->year, date_time_param->month); } else { snprintf(buffer, 30, "%d-%02d-%02d", date_time_param->year, date_time_param->month, date_time_param->day); } fname = buffer; fname = fname + DATAFILE_SUFFIX; ret_val = get_dir_name(device_id_param, date_time_param, period_type_param, calc_type_param); ret_val = FileUtil::concat_paths(ret_val, fname); return ret_val; } plp::Date *StoreCache::get_scanning_limit_date(EnumSummaryPeriod period_type_param) { Data *cur_data;; Date *ret_val; cur_data = get_latest_data(date, device_id, period_type_param); if (cur_data != NULL) { ret_val = cur_data->get_date().clone(); ret_val->next_second(); delete(cur_data); cur_data = NULL; } else { ret_val = date->clone(); if (period_type_param == PERIOD_YEARLY) { ret_val->next_year(); } else { ret_val->next_month(); } } return ret_val; } DataRange *StoreCache::get_mean(EnumSummaryPeriod period_type_param) { int row_count; DataRange *ret_val; Data *dta; string fname; ret_val = NULL; fname = get_file_name(device_id, date, period_type_param, MEAN); if (store_data == NULL) { if (access(fname.c_str(), R_OK) == 0) { load(fname); } } if (store_data != NULL) { row_count = store_data->get_count(); if (row_count > 0) { ret_val = new DataRange(); dta = store_data->get(0); ret_val->add(dta); delete(dta); } } if (ret_val == NULL) { switch(period_type_param) { case PERIOD_YEARLY: case PERIOD_MONTHLY: { Data *cur_data; Data *res_data; Date *cur_date; Date *max_date; int ii; int cnt; int val_cnt; DataRange *dr; Store *store; cur_date = date->clone(); max_date = get_scanning_limit_date(period_type_param); cur_data = NULL; res_data = NULL; cnt = 0; val_cnt = 0; dr = NULL; while(cur_date->before(max_date)) { if (period_type_param == PERIOD_YEARLY) { store = new StoreCache(device_id, cur_date); dr = store->get_mean(PERIOD_MONTHLY); } else { if (StoreDay::exist(device_id, cur_date, false)) { store = new StoreDay(device_id, cur_date); dr = store->get_mean(PERIOD_DAILY); } if (dr == NULL) { // data not found for that day, try to find next date containing data StoreDay::get_next_date_with_data(device_id, cur_date, max_date); continue; } } if (dr != NULL) { cur_data = dr->get_first(); if (cur_data != NULL) { cnt++; if (res_data == NULL) { res_data = cur_data; val_cnt = res_data->get_value_count(); } else { for (ii = 0; ii < val_cnt; ii++) { res_data->value_arr[ii] = res_data->value_arr[ii] + cur_data->value_arr[ii]; } delete(cur_data); } } delete(dr); } delete(store); if (period_type_param == PERIOD_YEARLY) { cur_date->next_month(); } else { cur_date->next_day(); } } if ((res_data != NULL) && (cnt > 0)) { for (ii = 0; ii < val_cnt; ii++) { res_data->value_arr[ii] = res_data->value_arr[ii] / cnt; } ret_val = new DataRange(res_data); save(fname, ret_val, 4); delete(res_data); } delete(cur_date); delete(max_date); } break; case PERIOD_DAILY: case PERIOD_HOURLY: case PERIOD_MINUTELY: case PERIOD_SECONDLY: if (StoreDay::exist(device_id, date, false)) { StoreDay *store; store = new StoreDay(device_id, date); ret_val = store->get_mean(period_type_param); if ((ret_val != NULL) && (period_type_param != PERIOD_MINUTELY) && (period_type_param != PERIOD_SECONDLY)) { // no need cache second or minute data save(fname, ret_val, 4); } delete(store); } break; } } return ret_val; } DataRange *StoreCache::get_sum(EnumSummaryPeriod period_type_param) { int row_count; DataRange *ret_val; Data *dta; string fname; Store *store; ret_val = NULL; fname = get_file_name(device_id, date, period_type_param, SUM); if (store_data == NULL) { if (access(fname.c_str(), R_OK) == 0) { load(fname); } } if (store_data != NULL) { row_count = store_data->get_count(); if (row_count > 0) { ret_val = new DataRange(); dta = store_data->get(0); ret_val->add(dta); delete(dta); } } if (ret_val == NULL) { switch(period_type_param) { case PERIOD_YEARLY: case PERIOD_MONTHLY: { Data *cur_data; Data *res_data; Date *cur_date; Date *max_date; int ii; int cnt; int val_cnt; DataRange *dr; cur_date = date->clone(); max_date = get_scanning_limit_date(period_type_param); cur_data = NULL; res_data = NULL; cnt = 0; while(cur_date->before(max_date)) { if (period_type_param == PERIOD_YEARLY) { store = new StoreCache(device_id, cur_date); dr = store->get_sum(PERIOD_MONTHLY); } else { if (StoreDay::exist(device_id, cur_date, false)) { store = new StoreDay(device_id, cur_date); dr = store->get_sum(PERIOD_DAILY); } if (dr == NULL) { // data not found for that day, try to find next date containing data StoreDay::get_next_date_with_data(device_id, cur_date, max_date); continue; } } if (dr != NULL) { cur_data = dr->get_first(); if (cur_data != NULL) { cnt++; if (res_data == NULL) { res_data = cur_data; } else { val_cnt = res_data->get_value_count(); for (ii = 0; ii < val_cnt; ii++) { res_data->value_arr[ii] = res_data->value_arr[ii] + cur_data->value_arr[ii]; } delete(cur_data); } } delete(dr); } delete(store); if (period_type_param == PERIOD_YEARLY) { cur_date->next_month(); } else { cur_date->next_day(); } } if ((res_data != NULL) && (cnt > 0)) { ret_val = new DataRange(res_data); save(fname, ret_val, 4); delete(res_data); } delete(cur_date); delete(max_date); } break; case PERIOD_DAILY: case PERIOD_HOURLY: case PERIOD_MINUTELY: case PERIOD_SECONDLY: if (StoreDay::exist(device_id, date, false)) { store = new StoreDay(device_id, date); ret_val = store->get_sum(period_type_param); if ((ret_val != NULL) && (period_type_param != PERIOD_MINUTELY) && (period_type_param != PERIOD_SECONDLY)) { // no need cache second or minute data save(fname, ret_val, 4); } delete(store); } break; } } return ret_val; } DataRange *StoreCache::get_delta(EnumSummaryPeriod period_type_param) { int row_count; DataRange *ret_val; Data *dta; string fname; StoreDay *store; ret_val = NULL; fname = get_file_name(device_id, date, period_type_param, DELTA); if (store_data == NULL) { if (access(fname.c_str(), R_OK) == 0) { // read from cache file load(fname); } } if (store_data != NULL) { row_count = store_data->get_count(); if (row_count > 0) { ret_val = new DataRange(); dta = store_data->get(0); ret_val->add(dta); delete(dta); } } if (ret_val == NULL) { switch(period_type_param) { case PERIOD_YEARLY: { Data *first_data; Data *last_data; int ii; int cnt; first_data = get_oldest_data(date, device_id, PERIOD_YEARLY); if (first_data != NULL) { last_data = get_latest_data(date, device_id, PERIOD_YEARLY); if (last_data != NULL) { ret_val = new DataRange(); cnt = last_data->get_value_count(); for (ii = 0; ii < cnt; ii++) { last_data->value_arr[ii] = last_data->value_arr[ii] - first_data->value_arr[ii]; } ret_val->add(last_data); delete(last_data); } delete(first_data); save(fname, ret_val, 4); } else { log_error("Could not read first or last data item from device %s for year: %d", device_id.c_str(), date->year); } } break; case PERIOD_MONTHLY: { Data *first_data; Data *last_data; Data *cur_data; Date *cur_date; Date *max_date; int ii; int cnt; cur_date = date->clone(); max_date = date->clone(); max_date->next_month(); first_data = NULL; last_data = NULL; while(cur_date->before(max_date)) { if (StoreDay::exist(device_id, cur_date, false)) { store = new StoreDay(device_id, cur_date); if (first_data == NULL) { cur_data = store->get_oldest_data(); if (cur_data != NULL) { first_data = cur_data->clone(); last_data = cur_data->clone(); delete(cur_data); } } cur_data = store->get_latest_data(); if (cur_data != NULL) { if (last_data != NULL) { delete(last_data); } last_data = cur_data; } delete(store); cur_date->next_day(); } else { // data not found for that day, try to find next date containing data StoreDay::get_next_date_with_data(device_id, cur_date, max_date); } } delete(cur_date); delete(max_date); if (first_data != NULL) { if (last_data == NULL) { last_data = first_data->clone(); } cnt = last_data->get_value_count(); for (ii = 0; ii < cnt; ii++) { last_data->value_arr[ii] = last_data->value_arr[ii] - first_data->value_arr[ii]; } cur_date = first_data->get_date().clone(); last_data->set_date(cur_date); delete(cur_date); ret_val = new DataRange(last_data); delete(first_data); delete(last_data); save(fname, ret_val, 4); } } break; case PERIOD_DAILY: case PERIOD_HOURLY: case PERIOD_MINUTELY: case PERIOD_SECONDLY: if (StoreDay::exist(device_id, date, false)) { store = new StoreDay(device_id, date); ret_val = store->get_delta(period_type_param); if ((period_type_param != PERIOD_MINUTELY) || (period_type_param != PERIOD_SECONDLY)) { // no need cache second or minute data save(fname, ret_val, 4); } delete(store); } break; } } return ret_val; } DataRange *StoreCache::get_max(EnumSummaryPeriod period_type_param) { int row_count; DataRange *ret_val; Data *dta; string fname; Store *store; ret_val = NULL; fname = get_file_name(device_id, date, period_type_param, MAX); if (store_data == NULL) { if (access(fname.c_str(), R_OK) == 0) { load(fname); } } if (store_data != NULL) { row_count = store_data->get_count(); if (row_count > 0) { ret_val = new DataRange(); dta = store_data->get(0); ret_val->add(dta); delete(dta); } } if (ret_val == NULL) { switch(period_type_param) { case PERIOD_YEARLY: case PERIOD_MONTHLY: { Data *cur_data; Data *res_data; Date *cur_date; Date *max_date; int ii; int cnt; int val_cnt; DataRange *dr; cur_date = date->clone(); max_date = get_scanning_limit_date(period_type_param); cur_data = NULL; res_data = NULL; cnt = 0; while(cur_date->before(max_date)) { if (period_type_param == PERIOD_YEARLY) { store = new StoreCache(device_id, cur_date); dr = store->get_max(PERIOD_MONTHLY); } else { if (StoreDay::exist(device_id, cur_date, false)) { store = new StoreDay(device_id, cur_date); dr = store->get_max(PERIOD_DAILY); } else { StoreDay::get_next_date_with_data(device_id, cur_date, max_date); continue; } } if (dr != NULL) { cur_data = dr->get_first(); if (cur_data != NULL) { cnt++; if (res_data == NULL) { res_data = cur_data; } else { val_cnt = res_data->get_value_count(); int changed = 0; for (ii = 0; ii < val_cnt; ii++) { if (cur_data->value_arr[ii] > res_data->value_arr[ii]) { res_data->value_arr[ii] = cur_data->value_arr[ii]; changed = 1; } } if (changed == 1) { Date new_date; new_date = cur_data->get_date(); res_data->set_date(&new_date); } delete(cur_data); } } delete(dr); } delete(store); if (period_type_param == PERIOD_YEARLY) { cur_date->next_month(); } else { cur_date->next_day(); } } if ((res_data != NULL) && (cnt > 0)) { ret_val = new DataRange(res_data); save(fname, ret_val, 4); delete(res_data); } delete(cur_date); delete(max_date); } break; case PERIOD_DAILY: case PERIOD_HOURLY: case PERIOD_MINUTELY: case PERIOD_SECONDLY: if (StoreDay::exist(device_id, date, false)) { store = new StoreDay(device_id, date); ret_val = store->get_max(period_type_param); if ((period_type_param != PERIOD_MINUTELY) || (period_type_param != PERIOD_SECONDLY)) { // no need cache second or minute data save(fname, ret_val, 4); } delete(store); } break; } } return ret_val; } DataRange *StoreCache::get_min(EnumSummaryPeriod period_type_param) { int row_count; DataRange *ret_val; Data *dta; string fname; Store *store; ret_val = NULL; fname = get_file_name(device_id, date, period_type_param, MIN); if (store_data == NULL) { if (access(fname.c_str(), R_OK) == 0) { load(fname); } } if (store_data != NULL) { row_count = store_data->get_count(); if (row_count > 0) { ret_val = new DataRange(); dta = store_data->get(0); ret_val->add(dta); delete(dta); } } if (ret_val == NULL) { switch(period_type_param) { case PERIOD_YEARLY: case PERIOD_MONTHLY: { Data *cur_data; Data *res_data; Date *cur_date; Date *max_date; int ii; int cnt; int val_cnt; DataRange *dr; cur_date = date->clone(); max_date = get_scanning_limit_date(period_type_param); cur_data = NULL; res_data = NULL; cnt = 0; while(cur_date->before(max_date)) { if (period_type_param == PERIOD_YEARLY) { store = new StoreCache(device_id, cur_date); dr = store->get_min(PERIOD_MONTHLY); } else { if (StoreDay::exist(device_id, cur_date, false)) { store = new StoreDay(device_id, cur_date); dr = store->get_min(PERIOD_DAILY); } else { StoreDay::get_next_date_with_data(device_id, cur_date, max_date); continue; } } if (dr != NULL) { cur_data = dr->get_first(); if (cur_data != NULL) { cnt++; if (res_data == NULL) { res_data = cur_data; } else { val_cnt = res_data->get_value_count(); int changed = 0; for (ii = 0; ii < val_cnt; ii++) { if (cur_data->value_arr[ii] < res_data->value_arr[ii]) { res_data->value_arr[ii] = cur_data->value_arr[ii]; changed = 1; } } if (changed == 1) { Date new_date; new_date = cur_data->get_date(); res_data->set_date(&new_date); } delete(cur_data); } } delete(dr); } delete(store); if (period_type_param == PERIOD_YEARLY) { cur_date->next_month(); } else { cur_date->next_day(); } } if ((res_data != NULL) && (cnt > 0)) { ret_val = new DataRange(res_data); save(fname, ret_val, 4); delete(res_data); } delete(cur_date); delete(max_date); } break; case PERIOD_DAILY: case PERIOD_HOURLY: case PERIOD_MINUTELY: case PERIOD_SECONDLY: if (StoreDay::exist(device_id, date, false)) { store = new StoreDay(device_id, date); ret_val = store->get_min(period_type_param); if ((period_type_param != PERIOD_MINUTELY) || (period_type_param != PERIOD_SECONDLY)) { // no need cache second or minute data save(fname, ret_val, 4); } delete(store); } break; } } return ret_val; } void StoreCache::save(std::string fname_param, plp::DataRange *datarange_param, int decimal_count_param) { string line; Data *data; ofstream *ostream; int ii; int cnt; cnt = datarange_param->get_count(); ostream = NULL; //log_info("[%s] cacheing %d data values.\n", device_id.c_str(), cnt); ostream = FileUtil::open_for_writing(fname_param.c_str()); if ((ostream != NULL) && (ostream->is_open() == true)) { // TODO: add mutex to protect string_list while it's read and emptied for(ii = 0; ii < cnt; ii++) { data = datarange_param->get(ii); if (data != NULL) { line = data->to_string(decimal_count_param); if (line.length() > 0) { *ostream << line << endl; } delete(data); } } } else { log_error("[%s] File open for data save failed: %s\n", device_id.c_str(), fname_param.c_str()); } if (ostream != NULL) { ostream->close(); delete(ostream); } } Data *StoreCache::get_oldest_data(Date *date_param, string device_id_param, EnumSummaryPeriod period_type_param) { int size; unsigned int ii; string year_dr; string mon_dr; vector mon_vcr; vector dta_vcr; string f_name; StoreDay *store; Data *ret_val; string device_dir; char buffer[30]; string year_name; ret_val = NULL; if (period_type_param == PERIOD_YEARLY) { snprintf(buffer, 30, "%d", date_param->year); year_name.append(buffer); device_dir = DeviceConfig::get_base_dir_name(); device_dir = FileUtil::concat_paths(device_dir, device_id_param); year_dr = FileUtil::concat_paths(device_dir, year_name); mon_vcr = FileUtil::get_subdirectories(year_dr); for (ii = 0; ii < mon_vcr.size(); ii++) { mon_dr = mon_vcr.at(ii); mon_dr = FileUtil::concat_paths(year_dr, mon_dr); // scan data files from month dir dta_vcr = FileUtil::get_data_files(mon_dr); size = dta_vcr.size(); if (size > 0) { f_name = dta_vcr.at(0); f_name = FileUtil::concat_paths(mon_dr, f_name); store = new StoreDay(f_name); ret_val = store->get_oldest_data(); delete(store); break; } } } else if (period_type_param == PERIOD_MONTHLY) { ret_val = NULL; snprintf(buffer, 30, "%d", date_param->year); year_name.append(buffer); device_dir = DeviceConfig::get_base_dir_name(); device_dir = FileUtil::concat_paths(device_dir, device_id_param); year_dr = FileUtil::concat_paths(device_dir, year_name); snprintf(buffer, 30, "%02d", date_param->month); mon_dr.append(buffer); mon_dr = FileUtil::concat_paths(year_dr, mon_dr); // scan data files from month dir dta_vcr = FileUtil::get_data_files(mon_dr); size = dta_vcr.size(); if (size > 0) { f_name = dta_vcr.at(0); f_name = FileUtil::concat_paths(mon_dr, f_name); store = new StoreDay(f_name); ret_val = store->get_oldest_data(); delete(store); } } else { store = new StoreDay(device_id_param, date_param); ret_val = store->get_oldest_data(); delete(store); } return ret_val; } Data *StoreCache::get_latest_data(Date *date_param, string device_id_param, EnumSummaryPeriod period_type_param) { int ii; string mon_dr; vector mon_vcr; vector dta_vcr; string f_name; StoreDay *store; Data *ret_val; string year_dr; int size; string device_dir; char buffer[30]; string year_name; ret_val = NULL; if (period_type_param == PERIOD_YEARLY) { snprintf(buffer, 30, "%d", date_param->year); year_name.append(buffer); device_dir = DeviceConfig::get_base_dir_name(); device_dir = FileUtil::concat_paths(device_dir, device_id_param); year_dr = FileUtil::concat_paths(device_dir, year_name); mon_vcr = FileUtil::get_subdirectories(year_dr); for (ii = mon_vcr.size() - 1; ii >= 0; ii--) { mon_dr = mon_vcr.at(ii); mon_dr = FileUtil::concat_paths(year_dr, mon_dr); // scan data files from month dir dta_vcr = FileUtil::get_data_files(mon_dr); size = dta_vcr.size(); if (size > 0) { f_name = dta_vcr.at(size - 1); f_name = FileUtil::concat_paths(mon_dr, f_name); store = new StoreDay(f_name); ret_val = store->get_latest_data(); delete(store); break; } } } else if (period_type_param == PERIOD_MONTHLY) { ret_val = NULL; snprintf(buffer, 30, "%d", date_param->year); year_name.append(buffer); device_dir = DeviceConfig::get_base_dir_name(); device_dir = FileUtil::concat_paths(device_dir, device_id_param); year_dr = FileUtil::concat_paths(device_dir, year_name); snprintf(buffer, 30, "%02d", date_param->month); mon_dr.append(buffer); mon_dr = FileUtil::concat_paths(year_dr, mon_dr); // scan data files from month dir dta_vcr = FileUtil::get_data_files(mon_dr); size = dta_vcr.size(); if (size > 0) { f_name = dta_vcr.at(size - 1); f_name = FileUtil::concat_paths(mon_dr, f_name); store = new StoreDay(f_name); ret_val = store->get_latest_data(); delete(store); } } else { store = new StoreDay(device_id_param, date_param); ret_val = store->get_latest_data(); delete(store); } return ret_val; }