XRootD
Loading...
Searching...
No Matches
XrdPfcConfiguration.cc
Go to the documentation of this file.
1#include "XrdPfc.hh"
2#include "XrdPfcTrace.hh"
3#include "XrdPfcInfo.hh"
4
5#include "XrdOss/XrdOss.hh"
6
7#include "XrdOuc/XrdOucEnv.hh"
11#include "XrdOuc/XrdOuca2x.hh"
12
14#include "XrdVersion.hh"
15
16#include <fcntl.h>
17
18using namespace XrdPfc;
19
21
23 m_hdfsmode(false),
24 m_allow_xrdpfc_command(false),
25 m_data_space("public"),
26 m_meta_space("public"),
27 m_diskTotalSpace(-1),
28 m_diskUsageLWM(-1),
29 m_diskUsageHWM(-1),
30 m_fileUsageBaseline(-1),
31 m_fileUsageNominal(-1),
32 m_fileUsageMax(-1),
33 m_purgeInterval(300),
34 m_purgeColdFilesAge(-1),
35 m_purgeAgeBasedPeriod(10),
36 m_accHistorySize(20),
37 m_dirStatsMaxDepth(-1),
38 m_dirStatsStoreDepth(0),
39 m_bufferSize(128*1024),
40 m_RamAbsAvailable(0),
41 m_RamKeepStdBlocks(0),
42 m_wqueue_blocks(16),
43 m_wqueue_threads(4),
44 m_prefetch_max_blocks(10),
45 m_hdfsbsize(128*1024*1024),
46 m_flushCnt(2000),
47 m_cs_UVKeep(-1),
48 m_cs_Chk(CSChk_Net),
49 m_cs_ChkTLS(false),
50 m_onlyIfCachedMinSize(1024*1024),
51 m_onlyIfCachedMinFrac(1.0)
52{}
53
54
55bool Cache::cfg2bytes(const std::string &str, long long &store, long long totalSpace, const char *name)
56{
57 char errStr[1024];
58 snprintf(errStr, 1024, "ConfigParameters() Error parsing parameter %s", name);
59
60 if (::isalpha(*(str.rbegin())))
61 {
62 if (XrdOuca2x::a2sz(m_log, errStr, str.c_str(), &store, 0, totalSpace))
63 {
64 return false;
65 }
66 }
67 else
68 {
69 char *eP;
70 errno = 0;
71 double frac = strtod(str.c_str(), &eP);
72 if (errno || eP == str.c_str())
73 {
74 m_log.Emsg(errStr, str.c_str());
75 return false;
76 }
77
78 store = static_cast<long long>(totalSpace * frac + 0.5);
79 }
80
81 if (store < 0 || store > totalSpace)
82 {
83 snprintf(errStr, 1024, "ConfigParameters() Error: parameter %s should be between 0 and total available disk space (%lld) - it is %lld (given as %s)",
84 name, totalSpace, store, str.c_str());
85 m_log.Emsg(errStr, "");
86 return false;
87 }
88
89 return true;
90}
91
92/* Function: xcschk
93
94 Purpose: To parse the directive: cschk <parms>
95
96 parms: [[no]net] [[no]tls] [[no]cache] [uvkeep <arg>]
97
98 all Checksum check on cache & net transfers.
99 cache Checksum check on cache only, 'no' turns it off.
100 net Checksum check on net transfers 'no' turns it off.
101 tls use TLS if server doesn't support checksums 'no' turns it off.
102 uvkeep Maximum amount of time a cached file make be kept if it
103 contains unverified checksums as n[d|h|m|s], where 'n'
104 is a non-negative integer. A value of 0 prohibits disk
105 caching unless the checksum can be verified. You can
106 also specify "lru" which means the standard purge policy
107 is to be used.
108
109 Output: true upon success or false upon failure.
110 */
111bool Cache::xcschk(XrdOucStream &Config)
112{
113 const char *val, *val2;
114 struct cschkopts {const char *opname; int opval;} csopts[] =
115 {
116 {"off", CSChk_None},
117 {"cache", CSChk_Cache},
118 {"net", CSChk_Net},
119 {"tls", CSChk_TLS}
120 };
121 int i, numopts = sizeof(csopts)/sizeof(struct cschkopts);
122 bool isNo;
123
124 if (! (val = Config.GetWord()))
125 {m_log.Emsg("Config", "cschk parameter not specified"); return false; }
126
127 while(val)
128 {
129 if ((isNo = strncmp(val, "no", 2) == 0))
130 val2 = val + 2;
131 else
132 val2 = val;
133 for (i = 0; i < numopts; i++)
134 {
135 if (!strcmp(val2, csopts[i].opname))
136 {
137 if (isNo)
138 m_configuration.m_cs_Chk &= ~csopts[i].opval;
139 else if (csopts[i].opval)
140 m_configuration.m_cs_Chk |= csopts[i].opval;
141 else
142 m_configuration.m_cs_Chk = csopts[i].opval;
143 break;
144 }
145 }
146 if (i >= numopts)
147 {
148 if (strcmp(val, "uvkeep"))
149 {
150 m_log.Emsg("Config", "invalid cschk option -", val);
151 return false;
152 }
153 if (!(val = Config.GetWord()))
154 {
155 m_log.Emsg("Config", "cschk uvkeep value not specified");
156 return false;
157 }
158 if (!strcmp(val, "lru"))
159 m_configuration.m_cs_UVKeep = -1;
160 else
161 {
162 int uvkeep;
163 if (XrdOuca2x::a2tm(m_log, "uvkeep time", val, &uvkeep, 0))
164 return false;
165 m_configuration.m_cs_UVKeep = uvkeep;
166 }
167 }
168 val = Config.GetWord();
169 }
170 // Decompose into separate TLS state, it is only passed on to psx
171 m_configuration.m_cs_ChkTLS = m_configuration.m_cs_Chk & CSChk_TLS;
172 m_configuration.m_cs_Chk &= ~CSChk_TLS;
173
174 m_env->Put("psx.CSNet", m_configuration.is_cschk_net() ? (m_configuration.m_cs_ChkTLS ? "2" : "1") : "0");
175
176 return true;
177}
178
179
180/* Function: xdlib
181
182 Purpose: To parse the directive: decisionlib <path> [<parms>]
183
184 <path> the path of the decision library to be used.
185 <parms> optional parameters to be passed.
186
187
188 Output: true upon success or false upon failure.
189 */
190bool Cache::xdlib(XrdOucStream &Config)
191{
192 const char* val;
193
194 std::string libp;
195 if (! (val = Config.GetWord()) || ! val[0])
196 {
197 TRACE(Info," Cache::Config() decisionlib not specified; always caching files");
198 return true;
199 }
200 else
201 {
202 libp = val;
203 }
204
205 char params[4096];
206 if (val[0])
207 Config.GetRest(params, 4096);
208 else
209 params[0] = 0;
210
211 XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "decisionlib",
212 libp.c_str());
213
214 Decision *(*ep)(XrdSysError&);
215 ep = (Decision *(*)(XrdSysError&))myLib->Resolve("XrdPfcGetDecision");
216 if (! ep) {myLib->Unload(true); return false; }
217
218 Decision * d = ep(m_log);
219 if (! d)
220 {
221 TRACE(Error, "Config() decisionlib was not able to create a decision object");
222 return false;
223 }
224 if (params[0])
225 d->ConfigDecision(params);
226
227 m_decisionpoints.push_back(d);
228 return true;
229}
230
231/* Function: xtrace
232
233 Purpose: To parse the directive: trace <level>
234 Output: true upon success or false upon failure.
235 */
236bool Cache::xtrace(XrdOucStream &Config)
237{
238 char *val;
239 static struct traceopts {const char *opname; int opval; } tropts[] =
240 {
241 {"none", 0},
242 {"error", 1},
243 {"warning", 2},
244 {"info", 3},
245 {"debug", 4},
246 {"dump", 5},
247 {"dumpxl", 6}
248 };
249 int numopts = sizeof(tropts)/sizeof(struct traceopts);
250
251 if (! (val = Config.GetWord()))
252 {m_log.Emsg("Config", "trace option not specified"); return 1; }
253
254 for (int i = 0; i < numopts; i++)
255 {
256 if (! strcmp(val, tropts[i].opname))
257 {
258 m_trace->What = tropts[i].opval;
259 return true;
260 }
261 }
262 m_log.Emsg("Config", "invalid trace option -", val);
263 return false;
264}
265
266
267//______________________________________________________________________________
268/* Function: Config
269
270 Purpose: To parse configuration file and configure Cache instance.
271 Output: true upon success or false upon failure.
272 */
273bool Cache::Config(const char *config_filename, const char *parameters)
274{
275 // Indicate whether or not we are a client instance
276 const char *theINS = getenv("XRDINSTANCE");
277 m_isClient = (theINS != 0 && strncmp("*client ", theINS, 8) == 0);
278
279 // Tell everyone else we are a caching proxy
280 XrdOucEnv::Export("XRDPFC", 1);
281
282 XrdOucEnv myEnv;
283 XrdOucStream Config(&m_log, theINS, &myEnv, "=====> ");
284
285 if (! config_filename || ! *config_filename)
286 {
287 TRACE(Error, "Config() configuration file not specified.");
288 return false;
289 }
290
291 int fd;
292 if ( (fd = open(config_filename, O_RDONLY, 0)) < 0)
293 {
294 TRACE( Error, "Config() can't open configuration file " << config_filename);
295 return false;
296 }
297
298 Config.Attach(fd);
299 static const char *cvec[] = { "*** pfc plugin config:", 0 };
300 Config.Capture(cvec);
301
302 // Obtain OFS configurator for OSS plugin.
303 XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log,
304 &XrdVERSIONINFOVAR(XrdOucGetCache));
305 if (! ofsCfg) return false;
306
307 TmpConfiguration tmpc;
308
309 // Adjust default parameters for client/serverless caching
310 if (m_isClient)
311 {
312 m_configuration.m_bufferSize = 128 * 1024; // same as normal.
313 m_configuration.m_wqueue_blocks = 8;
314 m_configuration.m_wqueue_threads = 1;
315 }
316
317 // If network checksum processing is the default, indicate so.
318 if (m_configuration.is_cschk_net()) m_env->Put("psx.CSNet", m_configuration.m_cs_ChkTLS ? "2" : "1");
319
320 // Actual parsing of the config file.
321 bool retval = true, aOK = true;
322 char *var;
323 while ((var = Config.GetMyFirstWord()))
324 {
325 if (! strcmp(var,"pfc.osslib"))
326 {
327 retval = ofsCfg->Parse(XrdOfsConfigPI::theOssLib);
328 }
329 else if (! strcmp(var,"pfc.cschk"))
330 {
331 retval = xcschk(Config);
332 }
333 else if (! strcmp(var,"pfc.decisionlib"))
334 {
335 retval = xdlib(Config);
336 }
337 else if (! strcmp(var,"pfc.trace"))
338 {
339 retval = xtrace(Config);
340 }
341 else if (! strcmp(var,"pfc.allow_xrdpfc_command"))
342 {
343 m_configuration.m_allow_xrdpfc_command = true;
344 }
345 else if (! strncmp(var,"pfc.", 4))
346 {
347 retval = ConfigParameters(std::string(var+4), Config, tmpc);
348 }
349
350 if ( ! retval)
351 {
352 TRACE(Error, "Config() error in parsing");
353 aOK = false;
354 }
355 }
356
357 Config.Close();
358
359 // Load OSS plugin.
360 myEnv.Put("oss.runmode", "pfc");
361 if (m_configuration.is_cschk_cache())
362 {
363 char csi_conf[128];
364 if (snprintf(csi_conf, 128, "space=%s nofill", m_configuration.m_meta_space.c_str()) < 128)
365 {
366 ofsCfg->Push(XrdOfsConfigPI::theOssLib, "libXrdOssCsi.so", csi_conf);
367 } else {
368 TRACE(Error, "Config() buffer too small for libXrdOssCsi params.");
369 return false;
370 }
371 }
372 if (ofsCfg->Load(XrdOfsConfigPI::theOssLib, &myEnv))
373 {
374 ofsCfg->Plugin(m_oss);
375 }
376 else
377 {
378 TRACE(Error, "Config() Unable to create an OSS object");
379 return false;
380 }
381
382 // sets default value for disk usage
383 XrdOssVSInfo sP;
384 {
385 if (m_oss->StatVS(&sP, m_configuration.m_data_space.c_str(), 1) < 0)
386 {
387 m_log.Emsg("ConfigParameters()", "error obtaining stat info for data space ", m_configuration.m_data_space.c_str());
388 return false;
389 }
390 if (sP.Total < 10ll << 20)
391 {
392 m_log.Emsg("ConfigParameters()", "available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ",
393 m_configuration.m_data_space.c_str());
394 return false;
395 }
396
397 m_configuration.m_diskTotalSpace = sP.Total;
398
399 if (cfg2bytes(tmpc.m_diskUsageLWM, m_configuration.m_diskUsageLWM, sP.Total, "lowWatermark") &&
400 cfg2bytes(tmpc.m_diskUsageHWM, m_configuration.m_diskUsageHWM, sP.Total, "highWatermark"))
401 {
402 if (m_configuration.m_diskUsageLWM >= m_configuration.m_diskUsageHWM) {
403 m_log.Emsg("ConfigParameters()", "pfc.diskusage should have lowWatermark < highWatermark.");
404 aOK = false;
405 }
406 }
407 else aOK = false;
408
409 if ( ! tmpc.m_fileUsageMax.empty())
410 {
411 if (cfg2bytes(tmpc.m_fileUsageBaseline, m_configuration.m_fileUsageBaseline, sP.Total, "files baseline") &&
412 cfg2bytes(tmpc.m_fileUsageNominal, m_configuration.m_fileUsageNominal, sP.Total, "files nominal") &&
413 cfg2bytes(tmpc.m_fileUsageMax, m_configuration.m_fileUsageMax, sP.Total, "files max"))
414 {
415 if (m_configuration.m_fileUsageBaseline >= m_configuration.m_fileUsageNominal ||
416 m_configuration.m_fileUsageBaseline >= m_configuration.m_fileUsageMax ||
417 m_configuration.m_fileUsageNominal >= m_configuration.m_fileUsageMax)
418 {
419 m_log.Emsg("ConfigParameters()", "pfc.diskusage files should have baseline < nominal < max.");
420 aOK = false;
421 }
422 }
423 else aOK = false;
424 }
425 }
426 // sets flush frequency
427 if ( ! tmpc.m_flushRaw.empty())
428 {
429 if (::isalpha(*(tmpc.m_flushRaw.rbegin())))
430 {
431 if (XrdOuca2x::a2sz(m_log, "Error getting number of bytes written before flush", tmpc.m_flushRaw.c_str(),
432 &m_configuration.m_flushCnt,
433 100 * m_configuration.m_bufferSize , 100000 * m_configuration.m_bufferSize))
434 {
435 return false;
436 }
437 m_configuration.m_flushCnt /= m_configuration.m_bufferSize;
438 }
439 else
440 {
441 if (XrdOuca2x::a2ll(m_log, "Error getting number of blocks written before flush", tmpc.m_flushRaw.c_str(),
442 &m_configuration.m_flushCnt, 100, 100000))
443 {
444 return false;
445 }
446 }
447 }
448
449 // get number of available RAM blocks after process configuration
450 if (m_configuration.m_RamAbsAvailable == 0)
451 {
452 m_configuration.m_RamAbsAvailable = m_isClient ? 256ll * 1024 * 1024 : 1024ll * 1024 * 1024;
453 char buff[1024];
454 snprintf(buff, sizeof(buff), "RAM usage pfc.ram is not specified. Default value %s is used.", m_isClient ? "256m" : "1g");
455 m_log.Say("Config info: ", buff);
456 }
457 // Setup number of standard-size blocks not released back to the system to 5% of total RAM.
458 m_configuration.m_RamKeepStdBlocks = (m_configuration.m_RamAbsAvailable / m_configuration.m_bufferSize + 1) * 5 / 100;
459
460
461 // Set tracing to debug if this is set in environment
462 char* cenv = getenv("XRDDEBUG");
463 if (cenv && ! strcmp(cenv,"1") && m_trace->What < 4) m_trace->What = 4;
464
465 if (aOK)
466 {
467 int loff = 0;
468// 000 001 010
469 const char *csc[] = {"off", "cache nonet", "nocache net notls",
470// 011
471 "cache net notls",
472// 100 101 110
473 "off", "cache nonet", "nocache net tls",
474// 111
475 "cache net tls"};
476 char buff[8192], uvk[32];
477 if (m_configuration.m_cs_UVKeep < 0)
478 strcpy(uvk, "lru");
479 else
480 sprintf(uvk, "%lld", (long long) m_configuration.m_cs_UVKeep);
481 float rg = (m_configuration.m_RamAbsAvailable) / float(1024*1024*1024);
482 loff = snprintf(buff, sizeof(buff), "Config effective %s pfc configuration:\n"
483 " pfc.cschk %s uvkeep %s\n"
484 " pfc.blocksize %lld\n"
485 " pfc.prefetch %d\n"
486 " pfc.ram %.fg\n"
487 " pfc.writequeue %d %d\n"
488 " # Total available disk: %lld\n"
489 " pfc.diskusage %lld %lld files %lld %lld %lld purgeinterval %d purgecoldfiles %d\n"
490 " pfc.spaces %s %s\n"
491 " pfc.trace %d\n"
492 " pfc.flush %lld\n"
493 " pfc.acchistorysize %d\n"
494 " pfc.onlyIfCachedMinBytes %lld\n"
495 " pfc.onlyIfCachedMinFrac %.2f\n",
496 config_filename,
497 csc[int(m_configuration.m_cs_Chk)], uvk,
498 m_configuration.m_bufferSize,
499 m_configuration.m_prefetch_max_blocks,
500 rg,
501 m_configuration.m_wqueue_blocks, m_configuration.m_wqueue_threads,
502 sP.Total,
503 m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM,
504 m_configuration.m_fileUsageBaseline, m_configuration.m_fileUsageNominal, m_configuration.m_fileUsageMax,
505 m_configuration.m_purgeInterval, m_configuration.m_purgeColdFilesAge,
506 m_configuration.m_data_space.c_str(),
507 m_configuration.m_meta_space.c_str(),
508 m_trace->What,
509 m_configuration.m_flushCnt,
510 m_configuration.m_accHistorySize,
511 m_configuration.m_onlyIfCachedMinSize,
512 m_configuration.m_onlyIfCachedMinFrac);
513
514 if (m_configuration.is_dir_stat_reporting_on())
515 {
516 loff += snprintf(buff + loff, sizeof(buff) - loff,
517 " pfc.dirstats maxdepth %d ((internal: store_depth %d, size_of_dirlist %d, size_of_globlist %d))\n",
518 m_configuration.m_dirStatsMaxDepth, m_configuration.m_dirStatsStoreDepth,
519 (int) m_configuration.m_dirStatsDirs.size(), (int) m_configuration.m_dirStatsDirGlobs.size());
520 loff += snprintf(buff + loff, sizeof(buff) - loff, " dirlist:\n");
521 for (std::set<std::string>::iterator i = m_configuration.m_dirStatsDirs.begin(); i != m_configuration.m_dirStatsDirs.end(); ++i)
522 loff += snprintf(buff + loff, sizeof(buff) - loff, " %s\n", i->c_str());
523 loff += snprintf(buff + loff, sizeof(buff) - loff, " globlist:\n");
524 for (std::set<std::string>::iterator i = m_configuration.m_dirStatsDirGlobs.begin(); i != m_configuration.m_dirStatsDirGlobs.end(); ++i)
525 loff += snprintf(buff + loff, sizeof(buff) - loff, " %s/*\n", i->c_str());
526 }
527
528 if (m_configuration.m_hdfsmode)
529 {
530 loff += snprintf(buff + loff, sizeof(buff) - loff, " pfc.hdfsmode hdfsbsize %lld\n", m_configuration.m_hdfsbsize);
531 }
532
533 if (m_configuration.m_username.empty())
534 {
535 char unameBuff[256];
536 XrdOucUtils::UserName(getuid(), unameBuff, sizeof(unameBuff));
537 m_configuration.m_username = unameBuff;
538 }
539 else
540 {
541 loff += snprintf(buff + loff, sizeof(buff) - loff, " pfc.user %s\n", m_configuration.m_username.c_str());
542 }
543
544 m_log.Say(buff);
545
546 m_env->Put("XRDPFC.SEGSIZE", std::to_string(m_configuration.m_bufferSize).c_str());
547 }
548
549 // Derived settings
550 m_prefetch_enabled = m_configuration.m_prefetch_max_blocks > 0;
551 Info::s_maxNumAccess = m_configuration.m_accHistorySize;
552
553 m_gstream = (XrdXrootdGStream*) m_env->GetPtr("pfc.gStream*");
554
555 m_log.Say("Config Proxy File Cache g-stream has", m_gstream ? "" : " NOT", " been configured via xrootd.monitor directive");
556
557 m_log.Say("------ Proxy File Cache configuration parsing ", aOK ? "completed" : "failed");
558
559 if (ofsCfg) delete ofsCfg;
560
561 // XXXX-CKSUM Testing. To be removed after OssPgi is also merged and valildated.
562 // Building of xrdpfc_print fails when this is enabled.
563#ifdef XRDPFC_CKSUM_TEST
564 {
565 int xxx = m_configuration.m_cs_Chk;
566
567 for (m_configuration.m_cs_Chk = CSChk_None; m_configuration.m_cs_Chk <= CSChk_Both; ++m_configuration.m_cs_Chk)
568 {
569 Info::TestCksumStuff();
570 }
571
572 m_configuration.m_cs_Chk = xxx;
573 }
574#endif
575
576 return aOK;
577}
578
579//------------------------------------------------------------------------------
580
581bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfiguration &tmpc)
582{
583 struct ConfWordGetter
584 {
585 XrdOucStream &m_config;
586 char *m_last_word;
587
588 ConfWordGetter(XrdOucStream& c) : m_config(c), m_last_word((char*)1) {}
589
590 const char* GetWord() { if (HasLast()) m_last_word = m_config.GetWord(); return HasLast() ? m_last_word : ""; }
591 bool HasLast() { return (m_last_word != 0); }
592 };
593
594 ConfWordGetter cwg(config);
595
596 XrdSysError err(0, "");
597 if ( part == "user" )
598 {
599 m_configuration.m_username = cwg.GetWord();
600 if ( ! cwg.HasLast())
601 {
602 m_log.Emsg("Config", "Error: pfc.user requires a parameter.");
603 return false;
604 }
605 }
606 else if ( part == "diskusage" )
607 {
608 tmpc.m_diskUsageLWM = cwg.GetWord();
609 tmpc.m_diskUsageHWM = cwg.GetWord();
610
611 if (tmpc.m_diskUsageHWM.empty())
612 {
613 m_log.Emsg("Config", "Error: pfc.diskusage parameter requires at least two arguments.");
614 return false;
615 }
616
617 const char *p = 0;
618 while ((p = cwg.GetWord()) && cwg.HasLast())
619 {
620 if (strcmp(p, "files") == 0)
621 {
622 tmpc.m_fileUsageBaseline = cwg.GetWord();
623 tmpc.m_fileUsageNominal = cwg.GetWord();
624 tmpc.m_fileUsageMax = cwg.GetWord();
625
626 if ( ! cwg.HasLast())
627 {
628 m_log.Emsg("Config", "Error: pfc.diskusage files directive requires three arguments.");
629 return false;
630 }
631 }
632 else if (strcmp(p, "sleep") == 0 || strcmp(p, "purgeinterval") == 0)
633 {
634 if (strcmp(p, "sleep") == 0) m_log.Emsg("Config", "warning sleep directive is deprecated in pfc.diskusage. Please use purgeinterval instead.");
635
636 if (XrdOuca2x::a2tm(m_log, "Error getting purgeinterval", cwg.GetWord(), &m_configuration.m_purgeInterval, 60, 3600))
637 {
638 return false;
639 }
640 }
641 else if (strcmp(p, "purgecoldfiles") == 0)
642 {
643 if (XrdOuca2x::a2tm(m_log, "Error getting purgecoldfiles age", cwg.GetWord(), &m_configuration.m_purgeColdFilesAge, 3600, 3600*24*360))
644 {
645 return false;
646 }
647 if (XrdOuca2x::a2i(m_log, "Error getting purgecoldfiles period", cwg.GetWord(), &m_configuration.m_purgeAgeBasedPeriod, 1, 1000))
648 {
649 return false;
650 }
651 }
652 else
653 {
654 m_log.Emsg("Config", "Error: diskusage stanza contains unknown directive", p);
655 }
656 }
657 }
658 else if ( part == "acchistorysize" )
659 {
660 if ( XrdOuca2x::a2i(m_log, "Error getting access-history-size", cwg.GetWord(), &m_configuration.m_accHistorySize, 20, 200))
661 {
662 return false;
663 }
664 }
665 else if ( part == "dirstats" )
666 {
667 const char *p = 0;
668 while ((p = cwg.GetWord()) && cwg.HasLast())
669 {
670 if (strcmp(p, "maxdepth") == 0)
671 {
672 if (XrdOuca2x::a2i(m_log, "Error getting maxdepth value", cwg.GetWord(), &m_configuration.m_dirStatsMaxDepth, 0, 16))
673 {
674 return false;
675 }
676 m_configuration.m_dirStatsStoreDepth = std::max(m_configuration.m_dirStatsStoreDepth, m_configuration.m_dirStatsMaxDepth);
677 }
678 else if (strcmp(p, "dir") == 0)
679 {
680 p = cwg.GetWord();
681 if (p && p[0] == '/')
682 {
683 // XXX -- should we just store them as sets of PathTokenizer objects, not strings?
684
685 char d[1024]; d[0] = 0;
686 int depth = 0;
687 { // Compress multiple slashes and "measure" depth
688 const char *pp = p;
689 char *pd = d;
690 *(pd++) = *(pp++);
691 while (*pp != 0)
692 {
693 if (*(pd - 1) == '/')
694 {
695 if (*pp == '/')
696 {
697 ++pp; continue;
698 }
699 ++depth;
700 }
701 *(pd++) = *(pp++);
702 }
703 *(pd--) = 0;
704 // remove trailing but but not leading /
705 if (*pd == '/' && pd != d) *pd = 0;
706 }
707 int ld = strlen(d);
708 if (ld >= 2 && d[ld-1] == '*' && d[ld-2] == '/')
709 {
710 d[ld-2] = 0;
711 ld -= 2;
712 m_configuration.m_dirStatsDirGlobs.insert(d);
713 printf("Glob %s -> %s -- depth = %d\n", p, d, depth);
714 }
715 else
716 {
717 m_configuration.m_dirStatsDirs.insert(d);
718 printf("Dir %s -> %s -- depth = %d\n", p, d, depth);
719 }
720
721 m_configuration.m_dirStatsStoreDepth = std::max(m_configuration.m_dirStatsStoreDepth, depth);
722 }
723 else
724 {
725 m_log.Emsg("Config", "Error: dirstats dir parameter requires a directory argument starting with a '/'.");
726 return false;
727 }
728 }
729 else
730 {
731 m_log.Emsg("Config", "Error: dirstats stanza contains unknown directive '", p, "'");
732 return false;
733 }
734 }
735 }
736 else if ( part == "blocksize" )
737 {
738 long long minBSize = 4 * 1024;
739 long long maxBSize = 512 * 1024 * 1024;
740 if (XrdOuca2x::a2sz(m_log, "Error reading block-size", cwg.GetWord(), &m_configuration.m_bufferSize, minBSize, maxBSize))
741 {
742 return false;
743 }
744 if (m_configuration.m_bufferSize & 0xFFF)
745 {
746 m_configuration.m_bufferSize &= ~0x0FFF;
747 m_configuration.m_bufferSize += 0x1000;
748 m_log.Emsg("Config", "pfc.blocksize must be a multiple of 4 kB. Rounded up.");
749 }
750 }
751 else if ( part == "prefetch" || part == "nramprefetch" )
752 {
753 if (part == "nramprefetch")
754 {
755 m_log.Emsg("Config", "pfc.nramprefetch is deprecated, please use pfc.prefetch instead. Replacing the directive internally.");
756 }
757
758 if (XrdOuca2x::a2i(m_log, "Error setting prefetch block count", cwg.GetWord(), &m_configuration.m_prefetch_max_blocks, 0, 128))
759 {
760 return false;
761 }
762
763 }
764 else if ( part == "nramread" )
765 {
766 m_log.Emsg("Config", "pfc.nramread is deprecated, please use pfc.ram instead. Ignoring this directive.");
767 cwg.GetWord(); // Ignoring argument.
768 }
769 else if ( part == "ram" )
770 {
771 long long minRAM = m_isClient ? 256 * 1024 * 1024 : 1024 * 1024 * 1024;
772 long long maxRAM = 256 * minRAM;
773 if ( XrdOuca2x::a2sz(m_log, "get RAM available", cwg.GetWord(), &m_configuration.m_RamAbsAvailable, minRAM, maxRAM))
774 {
775 return false;
776 }
777 }
778 else if ( part == "writequeue")
779 {
780 if (XrdOuca2x::a2i(m_log, "Error getting pfc.writequeue num-blocks", cwg.GetWord(), &m_configuration.m_wqueue_blocks, 1, 1024))
781 {
782 return false;
783 }
784 if (XrdOuca2x::a2i(m_log, "Error getting pfc.writequeue num-threads", cwg.GetWord(), &m_configuration.m_wqueue_threads, 1, 64))
785 {
786 return false;
787 }
788 }
789 else if ( part == "spaces" )
790 {
791 m_configuration.m_data_space = cwg.GetWord();
792 m_configuration.m_meta_space = cwg.GetWord();
793 if ( ! cwg.HasLast())
794 {
795 m_log.Emsg("Config", "spacenames requires two parameters: <data-space> <metadata-space>.");
796 return false;
797 }
798 }
799 else if ( part == "hdfsmode" )
800 {
801 m_log.Emsg("Config", "pfc.hdfsmode is currently unsupported.");
802 return false;
803
804 m_configuration.m_hdfsmode = true;
805
806 const char* params = cwg.GetWord();
807 if (params)
808 {
809 if (! strncmp("hdfsbsize", params, 9))
810 {
811 long long minBlSize = 32 * 1024;
812 long long maxBlSize = 128 * 1024 * 1024;
813 if ( XrdOuca2x::a2sz(m_log, "Error getting file fragment size", cwg.GetWord(), &m_configuration.m_hdfsbsize, minBlSize, maxBlSize))
814 {
815 return false;
816 }
817 }
818 else
819 {
820 m_log.Emsg("Config", "Error setting the fragment size parameter name");
821 return false;
822 }
823 }
824 }
825 else if ( part == "flush" )
826 {
827 tmpc.m_flushRaw = cwg.GetWord();
828 if ( ! cwg.HasLast())
829 {
830 m_log.Emsg("Config", "Error: pfc.flush requires a parameter.");
831 return false;
832 }
833 }
834 else if ( part == "onlyifcached" )
835 {
836 const char *p = 0;
837 while ((p = cwg.GetWord()) && cwg.HasLast())
838 {
839 if (strcmp(p, "minsize") == 0)
840 {
841 std::string minBytes = cwg.GetWord();
842 long long minBytesTop = 1024 * 1024 * 1024;
843 if (::isalpha(*(minBytes.rbegin())))
844 {
845 if (XrdOuca2x::a2sz(m_log, "Error in parsing minsize value for onlyifcached parameter", minBytes.c_str(), &m_configuration.m_onlyIfCachedMinSize, 0, minBytesTop))
846 {
847 return false;
848 }
849 }
850 else
851 {
852 if (XrdOuca2x::a2ll(m_log, "Error in parsing numeric minsize value for onlyifcached parameter", minBytes.c_str(),&m_configuration.m_onlyIfCachedMinSize, 0, minBytesTop))
853 {
854 return false;
855 }
856 }
857 }
858 if (strcmp(p, "minfrac") == 0)
859 {
860 std::string minFrac = cwg.GetWord();
861 char *eP;
862 errno = 0;
863 double frac = strtod(minFrac.c_str(), &eP);
864 if (errno || eP == minFrac.c_str())
865 {
866 m_log.Emsg("Config", "Error setting fraction for only-if-cached directive");
867 return false;
868 }
869 m_configuration.m_onlyIfCachedMinFrac = frac;
870 }
871 else
872 {
873 m_log.Emsg("Config", "Error: onlyifcached stanza contains unknown directive", p);
874 }
875 }
876 }
877 else
878 {
879 m_log.Emsg("ConfigParameters() unmatched pfc parameter", part.c_str());
880 return false;
881 }
882
883 return true;
884}
XrdVERSIONINFO(XrdOucGetCache, XrdPfc)
XrdOucCache * XrdOucGetCache(XrdSysLogger *logger, const char *config_filename, const char *parameters, XrdOucEnv *env)
Definition XrdPfc.cc:80
#define open
Definition XrdPosix.hh:71
int isNo(int dflt, const char *Msg1, const char *Msg2, const char *Msg3)
#define TRACE(act, x)
Definition XrdTrace.hh:63
bool Parse(TheLib what)
bool Plugin(XrdAccAuthorize *&piP)
Get Authorization plugin.
static XrdOfsConfigPI * New(const char *cfn, XrdOucStream *cfgP, XrdSysError *errP, XrdVersionInfo *verP=0, XrdSfsFileSystem *sfsP=0)
bool Load(int what, XrdOucEnv *envP=0)
bool Push(TheLib what, const char *plugP, const char *parmP=0)
@ theOssLib
Oss plugin.
long long Total
Definition XrdOssVS.hh:90
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
Definition XrdOss.cc:117
static int Export(const char *Var, const char *Val)
Definition XrdOucEnv.cc:170
void * GetPtr(const char *varname)
Definition XrdOucEnv.cc:263
void Put(const char *varname, const char *value)
Definition XrdOucEnv.hh:85
void * Resolve(const char *symbl, int mcnt=1)
void Unload(bool dodel=false)
char * GetWord(int lowcase=0)
static int UserName(uid_t uID, char *uName, int uNsz)
static int a2i(XrdSysError &, const char *emsg, const char *item, int *val, int minv=-1, int maxv=-1)
Definition XrdOuca2x.cc:45
static int a2sz(XrdSysError &, const char *emsg, const char *item, long long *val, long long minv=-1, long long maxv=-1)
Definition XrdOuca2x.cc:257
static int a2ll(XrdSysError &, const char *emsg, const char *item, long long *val, long long minv=-1, long long maxv=-1)
Definition XrdOuca2x.cc:70
static int a2tm(XrdSysError &, const char *emsg, const char *item, int *val, int minv=-1, int maxv=-1)
Definition XrdOuca2x.cc:288
bool Config(const char *config_filename, const char *parameters)
Parse configuration file.
Base class for selecting which files should be cached.
virtual bool ConfigDecision(const char *params)
Status of cached file. Can be read from and written into a binary file.
Definition XrdPfcInfo.hh:45
static size_t s_maxNumAccess
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
@ CSChk_Cache
long long m_hdfsbsize
used with m_hdfsmode, default 128MB
Definition XrdPfc.hh:108
long long m_RamAbsAvailable
available from configuration
Definition XrdPfc.hh:102
long long m_flushCnt
nuber of unsynced blcoks on disk before flush is called
Definition XrdPfc.hh:109
int m_accHistorySize
max number of entries in access history part of cinfo file
Definition XrdPfc.hh:94
int m_wqueue_threads
number of threads writing blocks to disk
Definition XrdPfc.hh:105
long long m_diskTotalSpace
total disk space on configured partition or oss space
Definition XrdPfc.hh:85
long long m_fileUsageMax
cache purge - files usage maximum
Definition XrdPfc.hh:90
long long m_fileUsageBaseline
cache purge - files usage baseline
Definition XrdPfc.hh:88
int m_dirStatsStoreDepth
depth to which statistics should be collected
Definition XrdPfc.hh:99
bool m_allow_xrdpfc_command
flag for enabling access to /xrdpfc-command/ functionality.
Definition XrdPfc.hh:79
long long m_diskUsageHWM
cache purge - disk usage high water mark
Definition XrdPfc.hh:87
bool is_cschk_cache() const
Definition XrdPfc.hh:69
std::set< std::string > m_dirStatsDirGlobs
directory globs for which stat reporting was requested
Definition XrdPfc.hh:97
int m_prefetch_max_blocks
maximum number of blocks to prefetch per file
Definition XrdPfc.hh:106
bool m_cs_ChkTLS
Allow TLS.
Definition XrdPfc.hh:113
long long m_fileUsageNominal
cache purge - files usage nominal
Definition XrdPfc.hh:89
int m_cs_Chk
Checksum check.
Definition XrdPfc.hh:112
int m_purgeAgeBasedPeriod
peform cold file / uvkeep purge every this many purge cycles
Definition XrdPfc.hh:93
bool m_hdfsmode
flag for enabling block-level operation
Definition XrdPfc.hh:78
int m_purgeColdFilesAge
purge files older than this age
Definition XrdPfc.hh:92
std::string m_data_space
oss space for data files
Definition XrdPfc.hh:82
std::set< std::string > m_dirStatsDirs
directories for which stat reporting was requested
Definition XrdPfc.hh:96
long long m_diskUsageLWM
cache purge - disk usage low water mark
Definition XrdPfc.hh:86
int m_RamKeepStdBlocks
number of standard-sized blocks kept after release
Definition XrdPfc.hh:103
long long m_bufferSize
prefetch buffer size, default 1MB
Definition XrdPfc.hh:101
std::string m_meta_space
oss space for metadata files (cinfo)
Definition XrdPfc.hh:83
int m_wqueue_blocks
maximum number of blocks written per write-queue loop
Definition XrdPfc.hh:104
std::string m_username
username passed to oss plugin
Definition XrdPfc.hh:81
bool is_cschk_net() const
Definition XrdPfc.hh:70
double m_onlyIfCachedMinFrac
minimum fraction of downloaded file, used by only-if-cached CGI option
Definition XrdPfc.hh:116
time_t m_cs_UVKeep
unverified checksum cache keep
Definition XrdPfc.hh:111
int m_dirStatsMaxDepth
maximum depth for statistics write out
Definition XrdPfc.hh:98
int m_purgeInterval
sleep interval between cache purges
Definition XrdPfc.hh:91
long long m_onlyIfCachedMinSize
minumum size of downloaded file, used by only-if-cached CGI option
Definition XrdPfc.hh:115
bool is_dir_stat_reporting_on() const
Definition XrdPfc.hh:62
std::string m_diskUsageLWM
Definition XrdPfc.hh:123
std::string m_diskUsageHWM
Definition XrdPfc.hh:124
std::string m_fileUsageBaseline
Definition XrdPfc.hh:125
std::string m_fileUsageNominal
Definition XrdPfc.hh:126
std::string m_flushRaw
Definition XrdPfc.hh:128
std::string m_fileUsageMax
Definition XrdPfc.hh:127