700 static const char *trc_pfx =
"Purge() ";
703 long long disk_usage;
704 long long estimated_file_usage = m_configuration.m_diskUsageHWM;
716 int age_based_purge_countdown = 0;
717 bool is_first =
true;
721 time_t purge_start = time(0);
732 long long bytesToRemove_d = 0, bytesToRemove_f = 0;
736 if (m_oss->StatVS(&sP, m_configuration.m_data_space.c_str(), 1) < 0)
738 TRACE(
Error, trc_pfx <<
"can't get StatVS for oss space " << m_configuration.m_data_space);
744 TRACE(
Debug, trc_pfx <<
"used disk space " << disk_usage <<
" bytes.");
746 if (disk_usage > m_configuration.m_diskUsageHWM)
748 bytesToRemove_d = disk_usage - m_configuration.m_diskUsageLWM;
753 if (m_configuration.are_file_usage_limits_set())
755 long long estimated_writes_since_last_purge;
759 estimated_writes_since_last_purge = m_writeQ.writes_between_purges;
760 m_writeQ.writes_between_purges = 0;
762 estimated_file_usage += estimated_writes_since_last_purge;
764 TRACE(
Debug, trc_pfx <<
"estimated usage by files " << estimated_file_usage <<
" bytes.");
766 bytesToRemove_f = std::max(estimated_file_usage - m_configuration.m_fileUsageNominal, 0ll);
769 double frac_du = 0, frac_fu = 0;
770 m_configuration.calculate_fractional_usages(disk_usage, estimated_file_usage, frac_du, frac_fu);
772 if (frac_fu > 1.0 - frac_du)
774 bytesToRemove_f = std::max(bytesToRemove_f, disk_usage - m_configuration.m_diskUsageLWM);
778 long long bytesToRemove = std::max(bytesToRemove_d, bytesToRemove_f);
780 bool enforce_age_based_purge =
false;
781 if (m_configuration.is_age_based_purge_in_effect() || m_configuration.is_uvkeep_purge_in_effect())
785 if (--age_based_purge_countdown <= 0)
787 enforce_age_based_purge =
true;
788 age_based_purge_countdown = m_configuration.m_purgeAgeBasedPeriod;
792 bool enforce_traversal_for_usage_collection = is_first;
795 copy_out_active_stats_and_update_data_fs_state();
798 TRACE(
Debug,
"\tbytes_to_remove_disk = " << bytesToRemove_d <<
" B");
799 TRACE(
Debug,
"\tbytes_to remove_files = " << bytesToRemove_f <<
" B (" << (is_first ?
"max possible for initial run" :
"estimated") <<
")");
800 TRACE(
Debug,
"\tbytes_to_remove = " << bytesToRemove <<
" B");
801 TRACE(
Debug,
"\tenforce_age_based_purge = " << enforce_age_based_purge);
804 long long bytesToRemove_at_start = 0;
805 int deleted_file_count = 0;
807 bool purge_required = (bytesToRemove > 0 || enforce_age_based_purge);
813 if (purge_required || enforce_traversal_for_usage_collection)
817 if (m_configuration.is_age_based_purge_in_effect())
819 purgeState.
setMinTime(time(0) - m_configuration.m_purgeColdFilesAge);
821 if (m_configuration.is_uvkeep_purge_in_effect())
826 XrdOssDF* dh = m_oss->newDir(m_configuration.m_username.c_str());
841 TRACE(
Debug, trc_pfx <<
"actual usage by files " << estimated_file_usage <<
" bytes.");
845 if (m_configuration.are_file_usage_limits_set())
847 bytesToRemove_f = std::max(estimated_file_usage - m_configuration.m_fileUsageNominal, 0ll);
849 double frac_du = 0, frac_fu = 0;
850 m_configuration.calculate_fractional_usages(disk_usage, estimated_file_usage, frac_du, frac_fu);
852 if (frac_fu > 1.0 - frac_du)
854 bytesToRemove = std::max(bytesToRemove_f, disk_usage - m_configuration.m_diskUsageLWM);
855 bytesToRemove = std::min(bytesToRemove, estimated_file_usage - m_configuration.m_fileUsageBaseline);
859 bytesToRemove = std::max(bytesToRemove_d, bytesToRemove_f);
864 bytesToRemove = std::max(bytesToRemove_d, bytesToRemove_f);
866 bytesToRemove_at_start = bytesToRemove;
869 TRACE(
Debug,
"\tbytes_to_remove_disk = " << bytesToRemove_d <<
" B");
870 TRACE(
Debug,
"\tbytes_to remove_files = " << bytesToRemove_f <<
" B (measured)");
871 TRACE(
Debug,
"\tbytes_to_remove = " << bytesToRemove <<
" B");
872 TRACE(
Debug,
"\tenforce_age_based_purge = " << enforce_age_based_purge);
875 if (enforce_age_based_purge)
883 if (m_configuration.is_dir_stat_reporting_on())
885 m_fs_state->dump_recursively();
893 int protected_cnt = 0;
894 long long protected_sum = 0;
899 if (bytesToRemove <= 0 && ! (enforce_age_based_purge && it->first == 0))
904 std::string &infoPath = it->second.path;
905 std::string dataPath = infoPath.substr(0, infoPath.size() - info_ext_len);
910 protected_sum += it->second.nBytes;
911 TRACE(
Debug, trc_pfx <<
"File is active or purge-protected: " << dataPath <<
" size: " << it->second.nBytes);
924 m_oss->Unlink(infoPath.c_str());
925 TRACE(Dump, trc_pfx <<
"Removed file: '" << infoPath <<
"' size: " <<
fstat.st_size);
931 bytesToRemove -= it->second.nBytes;
932 estimated_file_usage -= it->second.nBytes;
933 ++deleted_file_count;
935 m_oss->Unlink(dataPath.c_str());
936 TRACE(Dump, trc_pfx <<
"Removed file: '" << dataPath <<
"' size: " << it->second.nBytes <<
", time: " << it->first);
938 if (it->second.dirState != 0)
939 it->second.dirState->add_usage_purged(it->second.nBytes);
941 TRACE(
Error, trc_pfx <<
"DirState not set for file '" << dataPath <<
"'.");
944 if (protected_cnt > 0)
946 TRACE(
Info, trc_pfx <<
"Encountered " << protected_cnt <<
" protected files, sum of their size: " << protected_sum);
949 m_fs_state->upward_propagate_usage_purged();
955 m_purge_delay_set.clear();
959 int purge_duration = time(0) - purge_start;
961 TRACE(
Info, trc_pfx <<
"Finished, removed " << deleted_file_count <<
" data files, total size " <<
962 bytesToRemove_at_start - bytesToRemove <<
", bytes to remove at end " << bytesToRemove <<
", purge duration " << purge_duration);
964 int sleep_time = m_configuration.m_purgeInterval - purge_duration;