XRootD
Loading...
Searching...
No Matches
XrdClUtils.cc
Go to the documentation of this file.
1//------------------------------------------------------------------------------
2// Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN)
3// Author: Lukasz Janyst <ljanyst@cern.ch>
4//------------------------------------------------------------------------------
5// XRootD is free software: you can redistribute it and/or modify
6// it under the terms of the GNU Lesser General Public License as published by
7// the Free Software Foundation, either version 3 of the License, or
8// (at your option) any later version.
9//
10// XRootD is distributed in the hope that it will be useful,
11// but WITHOUT ANY WARRANTY; without even the implied warranty of
12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13// GNU General Public License for more details.
14//
15// You should have received a copy of the GNU Lesser General Public License
16// along with XRootD. If not, see <http://www.gnu.org/licenses/>.
17//------------------------------------------------------------------------------
18
19#include "XrdCl/XrdClUtils.hh"
25#include "XrdCl/XrdClMessage.hh"
26#include "XrdNet/XrdNetAddr.hh"
27
28#include <algorithm>
29#include <iomanip>
30#include <sstream>
31#include <fstream>
32#include <functional>
33#include <cctype>
34#include <locale>
35#include <map>
36#include <string>
37#include <set>
38#include <cctype>
39#include <random>
40#include <chrono>
41
42#include <sys/types.h>
43#include <dirent.h>
44
45#if __cplusplus < 201103L
46#include <ctime>
47#endif
48
49namespace
50{
51 bool isNotSpace( char c )
52 {
53 return c != ' ';
54 }
55
56 //----------------------------------------------------------------------------
57 // Ordering function for sorting IP addresses
58 //----------------------------------------------------------------------------
59 struct PreferIPv6
60 {
61 bool operator() ( const XrdNetAddr &l, const XrdNetAddr &r )
62 {
63 bool rIsIPv4 = false;
66 rIsIPv4 = true;
67
68 if( l.isIPType( XrdNetAddrInfo::IPv6 ) && rIsIPv4 )
69 return true;
70 return false;
71 }
72 };
73}
74
75namespace XrdCl
76{
77 //----------------------------------------------------------------------------
78 // Get a parameter either from the environment or URL
79 //----------------------------------------------------------------------------
80 int Utils::GetIntParameter( const URL &url,
81 const std::string &name,
82 int defaultVal )
83 {
84 Env *env = DefaultEnv::GetEnv();
85 int value = defaultVal;
86 char *endPtr;
87 URL::ParamsMap::const_iterator it;
88
89 env->GetInt( name, value );
90 it = url.GetParams().find( std::string("XrdCl.") + name );
91 if( it != url.GetParams().end() )
92 {
93 int urlValue = (int)strtol( it->second.c_str(), &endPtr, 0 );
94 if( !*endPtr )
95 value = urlValue;
96 }
97 return value;
98 }
99
100 //----------------------------------------------------------------------------
101 // Get a parameter either from the environment or URL
102 //----------------------------------------------------------------------------
103 std::string Utils::GetStringParameter( const URL &url,
104 const std::string &name,
105 const std::string &defaultVal )
106 {
107 Env *env = DefaultEnv::GetEnv();
108 std::string value = defaultVal;
109 URL::ParamsMap::const_iterator it;
110
111 env->GetString( name, value );
112 it = url.GetParams().find( std::string("XrdCl.") + name );
113 if( it != url.GetParams().end() )
114 value = it->second;
115
116 return value;
117 }
118
119 //----------------------------------------------------------------------------
120 // Interpret a string as address type, default to IPAll
121 //----------------------------------------------------------------------------
122 Utils::AddressType Utils::String2AddressType( const std::string &addressType )
123 {
124 if( addressType == "IPv6" )
125 return IPv6;
126 else if( addressType == "IPv4" )
127 return IPv4;
128 else if( addressType == "IPv4Mapped6" )
129 return IPv4Mapped6;
130 else if( addressType == "IPAll" )
131 return IPAll;
132 else
133 return IPAuto;
134 }
135
136 //----------------------------------------------------------------------------
137 // Resolve IP addresses
138 //----------------------------------------------------------------------------
139 Status Utils::GetHostAddresses( std::vector<XrdNetAddr> &addresses,
140 const URL &url,
141 Utils::AddressType type )
142 {
143 Log *log = DefaultEnv::GetLog();
144 const char *err = 0;
145 int ordn;
146
147 //--------------------------------------------------------------------------
148 // Resolve all the addresses
149 //--------------------------------------------------------------------------
150 std::ostringstream o; o << url.GetHostName() << ":" << url.GetPort();
152
153 if( type == IPv6 ) opts = XrdNetUtils::onlyIPv6;
154 else if( type == IPv4 ) opts = XrdNetUtils::onlyIPv4;
155 else if( type == IPv4Mapped6 ) opts = XrdNetUtils::allV4Map;
156 else if( type == IPAll ) opts = XrdNetUtils::allIPMap;
158
159 //--------------------------------------------------------------------------
160 // Check what are the preferences IPv6 or IPv4
161 //--------------------------------------------------------------------------
162 int preferIPv4 = DefaultPreferIPv4;
163 DefaultEnv::GetEnv()->GetInt( "PreferIPv4", preferIPv4 );
164
165 //--------------------------------------------------------------------------
166 // Partition the addresses according to the preferences
167 //
168 // The preferred IP family goes to the back as it is easier to remove
169 // items from the back of the vector
170 //--------------------------------------------------------------------------
172
173 //--------------------------------------------------------------------------
174 // Now get all of the properly partitioned addresses; ordn will hold the
175 // number of non-preferred addresses at the front of the table.
176 //--------------------------------------------------------------------------
177 err = XrdNetUtils::GetAddrs( o.str(), addresses, &ordn, opts );
178
179 if( err )
180 {
181 log->Error( UtilityMsg, "Unable to resolve %s: %s", o.str().c_str(),
182 err );
183 return Status( stError, errInvalidAddr );
184 }
185
186 if( addresses.size() == 0 )
187 {
188 log->Error( UtilityMsg, "No addresses for %s were found",
189 o.str().c_str() );
190 return Status( stError, errInvalidAddr );
191 }
192
193 //--------------------------------------------------------------------------
194 // Shuffle each partition
195 //--------------------------------------------------------------------------
196
197 int ipNoShuffle = DefaultIPNoShuffle;
198 Env *env = DefaultEnv::GetEnv();
199 env->GetInt( "IPNoShuffle", ipNoShuffle );
200
201 if( !ipNoShuffle )
202 {
203#if __cplusplus < 201103L
204 // initialize the random generator only once
205 static struct only_once_t
206 {
207 only_once_t()
208 {
209 std::srand ( unsigned ( std::time(0) ) );
210 }
211 } only_once;
212
213 std::random_shuffle( addresses.begin(), addresses.begin() + ordn );
214 std::random_shuffle( addresses.begin() + ordn, addresses.end() );
215#else
216 static std::default_random_engine rand_engine(
217 std::chrono::system_clock::now().time_since_epoch().count() );
218
219 std::shuffle( addresses.begin(), addresses.begin() + ordn, rand_engine );
220 std::shuffle( addresses.begin() + ordn, addresses.end(), rand_engine );
221#endif
222 }
223
224 //--------------------------------------------------------------------------
225 // Return status as the result is already in the output parameter
226 //--------------------------------------------------------------------------
227 return Status();
228 }
229
230 //----------------------------------------------------------------------------
231 // Log all the addresses on the list
232 //----------------------------------------------------------------------------
234 uint64_t type,
235 const std::string &hostId,
236 std::vector<XrdNetAddr> &addresses )
237 {
238 std::string addrStr;
239 std::vector<XrdNetAddr>::iterator it;
240 for( it = addresses.begin(); it != addresses.end(); ++it )
241 {
242 char nameBuff[256];
243 it->Format( nameBuff, 256, XrdNetAddrInfo::fmtAdv6 );
244 addrStr += nameBuff;
245 addrStr += ", ";
246 }
247 addrStr.erase( addrStr.length()-2, 2 );
248 log->Debug( type, "[%s] Found %d address(es): %s",
249 hostId.c_str(), addresses.size(), addrStr.c_str() );
250 }
251
252 //----------------------------------------------------------------------------
253 // Convert timestamp to a string
254 //----------------------------------------------------------------------------
255 std::string Utils::TimeToString( time_t timestamp )
256 {
257 char now[30];
258 tm tsNow;
259 time_t ttNow = timestamp;
260 localtime_r( &ttNow, &tsNow );
261 strftime( now, 30, "%Y-%m-%d %H:%M:%S %z", &tsNow );
262 return now;
263 }
264
265 //----------------------------------------------------------------------------
266 // Get the elapsed microseconds between two timevals
267 //----------------------------------------------------------------------------
268 uint64_t Utils::GetElapsedMicroSecs( timeval start, timeval end )
269 {
270 uint64_t startUSec = start.tv_sec*1000000 + start.tv_usec;
271 uint64_t endUSec = end.tv_sec*1000000 + end.tv_usec;
272 return endUSec-startUSec;
273 }
274
275 //----------------------------------------------------------------------------
276 // Get remote checksum
277 //----------------------------------------------------------------------------
279 const std::string &checkSumType,
280 const URL &url )
281 {
282 FileSystem *fs = new FileSystem( url );
283 // add the 'cks.type' cgi tag in order to
284 // select the proper checksum type in case
285 // the server supports more than one checksum
286 size_t pos = url.GetPath().find( '?' );
287 std::string cksPath = url.GetPath() + ( pos == std::string::npos ? '?' : '&' ) + "cks.type=" + checkSumType;
288 Buffer arg; arg.FromString( cksPath );
289 Buffer *cksResponse = 0;
290 XRootDStatus st;
291 Log *log = DefaultEnv::GetLog();
292
293 st = fs->Query( QueryCode::Checksum, arg, cksResponse );
294 delete fs;
295
296 if( !st.IsOK() )
297 {
298 std::string msg = st.GetErrorMessage();
299 msg += " Got an error while querying the checksum!";
300 st.SetErrorMessage( msg );
301 return st;
302 }
303
304 if( !cksResponse )
305 return XRootDStatus( stError, errInternal, 0, "Got invalid response while querying the checksum!" );
306
307 std::vector<std::string> elems;
308 Utils::splitString( elems, cksResponse->ToString(), " " );
309 delete cksResponse;
310
311 if( elems.size() != 2 )
312 return XRootDStatus( stError, errInvalidResponse, 0, "Got invalid response while querying the checksum!" );
313
314 if( elems[0] != checkSumType )
316
317 checkSum = elems[0] + ":";
318 checkSum += NormalizeChecksum( elems[0], elems[1] );
319
320 log->Dump( UtilityMsg, "Checksum for %s checksum: %s",
321 url.GetPath().c_str(), checkSum.c_str() );
322
323 return XRootDStatus();
324 }
325
326 //------------------------------------------------------------------------
327 // Get a checksum from local file
328 //------------------------------------------------------------------------
329 XRootDStatus Utils::GetLocalCheckSum( std::string &checkSum,
330 const std::string &checkSumType,
331 const std::string &path )
332 {
333 Log *log = DefaultEnv::GetLog();
335
336 if( !cksMan )
337 {
338 log->Error( UtilityMsg, "Unable to get the checksum manager" );
340 }
341
342 XrdCksData ckSum; ckSum.Set( checkSumType.c_str() );
343 bool status = cksMan->Calculate( ckSum, checkSumType, path.c_str() );
344 if( !status )
345 {
346 log->Error( UtilityMsg, "Error while calculating checksum for %s",
347 path.c_str() );
349 }
350
351 char *cksBuffer = new char[265];
352 ckSum.Get( cksBuffer, 256 );
353 checkSum = checkSumType + ":";
354 checkSum += NormalizeChecksum( checkSumType, cksBuffer );
355 delete [] cksBuffer;
356
357 log->Dump( UtilityMsg, "Checksum for %s is: %s", path.c_str(),
358 checkSum.c_str() );
359
360 return XRootDStatus();
361 }
362
363 //----------------------------------------------------------------------------
364 // Convert bytes to a human readable string
365 //----------------------------------------------------------------------------
366 std::string Utils::BytesToString( uint64_t bytes )
367 {
368 double final = bytes;
369 int i = 0;
370 char suf[3] = { 'k', 'M', 'G' };
371 for( i = 0; i < 3 && final > 1024; ++i, final /= 1024 ) {};
372 std::ostringstream o;
373 o << std::setprecision(4) << final;
374 if( i > 0 ) o << suf[i-1];
375 return o.str();
376 }
377
378 //----------------------------------------------------------------------------
379 // Check if peer supports tpc
380 //----------------------------------------------------------------------------
381 XRootDStatus Utils::CheckTPC( const std::string &server, uint16_t timeout )
382 {
383 Log *log = DefaultEnv::GetLog();
384 log->Debug( UtilityMsg, "Checking if the data server %s supports tpc",
385 server.c_str() );
386
387 FileSystem sourceDSFS( server );
388 Buffer queryArg; queryArg.FromString( "tpc" );
389 Buffer *queryResponse = 0;
390 XRootDStatus st;
391 st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
392 timeout );
393 if( !st.IsOK() )
394 {
395 log->Error( UtilityMsg, "Cannot query source data server %s: %s",
396 server.c_str(), st.ToStr().c_str() );
397 st.status = stFatal;
398 return st;
399 }
400
401 if( !queryResponse )
402 {
403 log->Error( UtilityMsg, "Cannot query source data server: empty response." );
404 st.status = stFatal;
405 return st;
406 }
407
408 std::string answer = queryResponse->ToString();
409 delete queryResponse;
410 if( answer.length() == 1 || !isdigit( answer[0] ) || atoi(answer.c_str()) == 0)
411 {
412 log->Debug( UtilityMsg, "Third party copy not supported at: %s",
413 server.c_str() );
415 }
416 log->Debug( UtilityMsg, "Third party copy supported at: %s",
417 server.c_str() );
418
419 return XRootDStatus();
420 }
421
422 //------------------------------------------------------------------------
423 // Check if peer supports tpc / tpc lite
424 //------------------------------------------------------------------------
425 XRootDStatus Utils::CheckTPCLite( const std::string &server, uint16_t timeout )
426 {
427 Log *log = DefaultEnv::GetLog();
428 log->Debug( UtilityMsg, "Checking if the data server %s supports tpc / tpc lite",
429 server.c_str() );
430
431 FileSystem sourceDSFS( server );
432 Buffer queryArg; queryArg.FromString( "tpc tpcdlg" );
433 Buffer *queryResponse = 0;
434 XRootDStatus st;
435 st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
436 timeout );
437 if( !st.IsOK() )
438 {
439 log->Error( UtilityMsg, "Cannot query source data server %s: %s",
440 server.c_str(), st.ToStr().c_str() );
441 st.status = stFatal;
442 return st;
443 }
444
445 if( !queryResponse )
446 {
447 log->Error( UtilityMsg, "Cannot query source data server: empty response." );
448 st.status = stFatal;
449 return st;
450 }
451
452 std::string answer = queryResponse->ToString();
453 delete queryResponse;
454
455 if( answer.empty() )
456 {
457 log->Error( UtilityMsg, "Cannot query source data server: empty response." );
458 st.status = stFatal;
459 return st;
460 }
461
462 std::vector<std::string> resp;
463 Utils::splitString( resp, answer, "\n" );
464
465 if( resp.empty() || resp[0].empty() ||
466 !isdigit( resp[0][0]) || atoi( resp[0].c_str() ) == 0 )
467 {
468 log->Debug( UtilityMsg, "Third party copy not supported at: %s",
469 server.c_str() );
471 }
472
473 if( resp.size() == 1 || resp[1] == "tpcdlg" )
474 {
475 log->Debug( UtilityMsg, "TPC lite not supported at: %s",
476 server.c_str() );
477 return XRootDStatus( stOK, suPartial );
478 }
479
480 log->Debug( UtilityMsg, "TPC lite supported at: %s",
481 server.c_str() );
482
483 return XRootDStatus();
484 }
485
486 //----------------------------------------------------------------------------
487 // Convert the fully qualified host name to country code
488 //----------------------------------------------------------------------------
489 std::string Utils::FQDNToCC( const std::string &fqdn )
490 {
491 std::vector<std::string> el;
492 Utils::splitString( el, fqdn, "." );
493 if( el.size() < 2 )
494 return "us";
495
496 std::string cc = *el.rbegin();
497 if( cc.length() == 2 )
498 return cc;
499 return "us";
500 }
501
502 //----------------------------------------------------------------------------
503 // Get directory entries
504 //----------------------------------------------------------------------------
505 Status Utils::GetDirectoryEntries( std::vector<std::string> &entries,
506 const std::string &path )
507 {
508 DIR *dp = opendir( path.c_str() );
509 if( !dp )
510 return Status( stError, errOSError, errno );
511
512 dirent *dirEntry;
513
514 while( (dirEntry = readdir(dp)) != 0 )
515 {
516 std::string entryName = dirEntry->d_name;
517 if( !entryName.compare( 0, 2, "..") )
518 continue;
519 if( !entryName.compare( 0, 1, ".") )
520 continue;
521
522 entries.push_back( dirEntry->d_name );
523 }
524
525 closedir(dp);
526
527 return Status();
528 }
529
530 //----------------------------------------------------------------------------
531 // Process a config file and return key-value pairs
532 //----------------------------------------------------------------------------
533 Status Utils::ProcessConfig( std::map<std::string, std::string> &config,
534 const std::string &file )
535 {
536 config.clear();
537 std::ifstream inFile( file.c_str() );
538 if( !inFile.good() )
539 return Status( stError, errOSError, errno );
540
541 errno = 0;
542 std::string line;
543 while( std::getline( inFile, line ) )
544 {
545 if( line.empty() || line[0] == '#' )
546 continue;
547
548 std::vector<std::string> elems;
549 splitString( elems, line, "=" );
550 if( elems.size() != 2 )
551 return Status( stError, errConfig );
552 std::string key = elems[0]; Trim( key );
553 std::string value = elems[1]; Trim( value );
554 config[key] = value;
555 }
556
557 if( errno )
558 return Status( stError, errOSError, errno );
559 return Status();
560 }
561
562 //------------------------------------------------------------------------
564 //------------------------------------------------------------------------
565 Status Utils::ProcessConfigDir( std::map<std::string, std::string> &config,
566 const std::string &dir )
567 {
568 Log *log = DefaultEnv::GetLog();
569 log->Debug( UtilityMsg, "Processing configuration files in %s...",
570 dir.c_str());
571
572 std::vector<std::string> entries;
573 Status st = Utils::GetDirectoryEntries( entries, dir );
574 if( !st.IsOK() )
575 {
576 log->Debug( UtilityMsg, "Unable to process directory %s: %s",
577 dir.c_str(), st.ToString().c_str() );
578 return st;
579 }
580
581 static const std::string suffix = ".conf";
582 for( auto &entry : entries )
583 {
584 std::string confFile = dir + "/" + entry;
585
586 if( confFile.length() <= suffix.length() )
587 continue;
588 if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) )
589 continue;
590
591 st = ProcessConfig( config, confFile );
592 if( !st.IsOK() )
593 {
594 log->Debug( UtilityMsg, "Unable to process configuration file %s: %s",
595 confFile.c_str(), st.ToString().c_str() );
596 }
597 }
598
599 return Status();
600 }
601
602 //----------------------------------------------------------------------------
603 // Trim a string
604 //----------------------------------------------------------------------------
605 void Utils::Trim( std::string &str )
606 {
607 str.erase( str.begin(),
608 std::find_if( str.begin(), str.end(), isNotSpace ) );
609 str.erase( std::find_if( str.rbegin(), str.rend(), isNotSpace ).base(),
610 str.end() );
611 }
612
613 //----------------------------------------------------------------------------
614 // Log property list
615 //----------------------------------------------------------------------------
617 uint64_t topic,
618 const char *format,
619 const PropertyList &list )
620 {
621 PropertyList::PropertyMap::const_iterator it;
622 std::string keyVals;
623 for( it = list.begin(); it != list.end(); ++it )
624 keyVals += "'" + it->first + "' = '" + it->second + "', ";
625 keyVals.erase( keyVals.length()-2, 2 );
626 log->Dump( topic, format, keyVals.c_str() );
627 }
628
629 //----------------------------------------------------------------------------
630 // Print a char array as hex
631 //----------------------------------------------------------------------------
632 std::string Utils::Char2Hex( uint8_t *array, uint16_t size )
633 {
634 char *hex = new char[2*size+1];
635 for( uint16_t i = 0; i < size; ++i )
636 snprintf( hex+(2*i), 3, "%02x", (int)array[i] );
637 std::string result = hex;
638 delete [] hex;
639 return result;
640 }
641
642 //----------------------------------------------------------------------------
643 // Normalize checksum
644 //----------------------------------------------------------------------------
645 std::string Utils::NormalizeChecksum( const std::string &name,
646 const std::string &checksum )
647 {
648 if( name == "adler32" || name == "crc32" )
649 {
650 size_t i;
651 for( i = 0; i < checksum.length(); ++i )
652 if( checksum[i] != '0' )
653 break;
654 return checksum.substr(i);
655 }
656 return checksum;
657 }
658
659 //----------------------------------------------------------------------------
660 // Get supported checksum types for given URL
661 //----------------------------------------------------------------------------
662 std::vector<std::string> Utils::GetSupportedCheckSums( const XrdCl::URL &url )
663 {
664 std::vector<std::string> ret;
665
666 FileSystem fs( url );
667 Buffer arg; arg.FromString( "chksum" );
668 Buffer *resp = 0;
669 XRootDStatus st = fs.Query( QueryCode::Config, arg, resp );
670 if( st.IsOK() )
671 {
672 std::string response = resp->ToString();
673 if( response != "chksum" )
674 {
675 // we are expecting a response of format: '0:zcrc32,1:adler32'
676 std::vector<std::string> result;
677 Utils::splitString( result, response, "," );
678
679 std::vector<std::string>::iterator itr = result.begin();
680 for( ; itr != result.end(); ++itr )
681 {
682 size_t pos = itr->find( ':' );
683 if( pos == std::string::npos ) continue;
684 std::string cksname = itr->substr( pos + 1 );
685 // remove all white spaces
686 cksname.erase( std::remove_if( cksname.begin(), cksname.end(), ::isspace ),
687 cksname.end() );
688 ret.push_back( std::move( cksname ) );
689 }
690 }
691 }
692
693 return ret;
694 }
695
696
697 //------------------------------------------------------------------------
699 //------------------------------------------------------------------------
700 bool Utils::CheckEC( const Message *req, const URL &url )
701 {
702#ifdef WITH_XRDEC
703 // make sure that if we will be writing it is a new file
704 ClientRequest *request = (ClientRequest*)req->GetBuffer();
705 uint16_t options = ntohs( request->open.options );
706 bool open_wrt = ( options & kXR_open_updt ) || ( options & kXR_open_wrto );
707 bool open_new = ( options & kXR_new );
708 if( open_wrt && !open_new ) return false;
709
710 const URL::ParamsMap &params = url.GetParams();
711 // make sure all the xrdec. tokens are present and the values are sane
712 URL::ParamsMap::const_iterator itr = params.find( "xrdec.nbdta" );
713 if( itr == params.end() ) return false;
714 size_t nbdta = std::stoul( itr->second );
715
716 itr = params.find( "xrdec.nbprt" );
717 if( itr == params.end() ) return false;
718 size_t nbprt = std::stoul( itr->second );
719
720 itr = params.find( "xrdec.blksz" );
721 if( itr == params.end() ) return false;
722
723 itr = params.find( "xrdec.plgr" );
724 if( itr == params.end() ) return false;
725 std::vector<std::string> plgr;
726 splitString( plgr, itr->second, "," );
727 if( plgr.size() < nbdta + nbprt ) return false;
728
729 itr = params.find( "xrdec.objid" );
730 if( itr == params.end() ) return false;
731
732 itr = params.find( "xrdec.format" );
733 if( itr == params.end() ) return false;
734 size_t format = std::stoul( itr->second );
735 if( format != 1 ) return false; // TODO use constant
736
737 itr = params.find( "xrdec.dtacgi" );
738 if( itr != params.end() )
739 {
740 std::vector<std::string> dtacgi;
741 splitString( dtacgi, itr->second, "," );
742 if( plgr.size() != dtacgi.size() ) return false;
743 }
744
745 itr = params.find( "xrdec.mdtacgi" );
746 if( itr != params.end() )
747 {
748 std::vector<std::string> mdtacgi;
749 splitString( mdtacgi, itr->second, "," );
750 if( plgr.size() != mdtacgi.size() ) return false;
751 }
752
753 itr = params.find( "xrdec.cosc" );
754 if( itr == params.end() ) return false;
755 std::string cosc = itr->second;
756 if( cosc != "true" && cosc != "false" ) return false;
757
758 return true;
759#else
760 return false;
761#endif
762 }
763
764
765 //----------------------------------------------------------------------------
767 //----------------------------------------------------------------------------
768 std::string Utils::InferChecksumType( const XrdCl::URL &source,
769 const XrdCl::URL &destination,
770 bool zip)
771 {
772 //--------------------------------------------------------------------------
773 // If both files are local we won't be checksumming at all
774 //--------------------------------------------------------------------------
775 if( source.IsLocalFile() && !source.IsMetalink() && destination.IsLocalFile() ) return std::string();
776
777 // checksums supported by local files
778 std::set<std::string> local_supported;
779 local_supported.insert( "adler32" );
780 local_supported.insert( "crc32" );
781 local_supported.insert( "md5" );
782 local_supported.insert( "zcrc32" );
783
784 std::vector<std::string> srccks;
785
786 if( source.IsMetalink() )
787 {
788 int useMtlnCksum = DefaultZipMtlnCksum;
789 Env *env = DefaultEnv::GetEnv();
790 env->GetInt( "ZipMtlnCksum", useMtlnCksum );
791
792 //------------------------------------------------------------------------
793 // In case of ZIP use other checksums than zcrc32 only if the user
794 // requested it explicitly.
795 //------------------------------------------------------------------------
796 if( !zip || ( zip && useMtlnCksum ) )
797 {
799 VirtualRedirector *redirector = registry.Get( source );
800 std::vector<std::string> cks = redirector->GetSupportedCheckSums();
801 srccks.insert( srccks.end(), cks.begin(), cks.end() );
802 }
803 }
804
805 if( zip )
806 {
807 //------------------------------------------------------------------------
808 // In case of ZIP we can always extract the checksum from the archive
809 //------------------------------------------------------------------------
810 srccks.push_back( "zcrc32" );
811 }
812 else if( source.GetProtocol() == "root" || source.GetProtocol() == "xroot" )
813 {
814 //------------------------------------------------------------------------
815 // If the source is a remote endpoint query the supported checksums
816 //------------------------------------------------------------------------
817 std::vector<std::string> cks = GetSupportedCheckSums( source );
818 srccks.insert( srccks.end(), cks.begin(), cks.end() );
819 }
820
821 std::vector<std::string> dstcks;
822
823 if( destination.GetProtocol() == "root" ||
824 destination.GetProtocol() == "xroot" )
825 {
826 //------------------------------------------------------------------------
827 // If the destination is a remote endpoint query the supported checksums
828 //------------------------------------------------------------------------
829 std::vector<std::string> cks = GetSupportedCheckSums( destination );
830 dstcks.insert( dstcks.end(), cks.begin(), cks.end() );
831 }
832
833 //--------------------------------------------------------------------------
834 // Now we have all the information we need, we can infer the right checksum
835 // type!!!
836 //
837 // First check if source is local
838 //--------------------------------------------------------------------------
839 if( source.IsLocalFile() && !source.IsMetalink() )
840 {
841 std::vector<std::string>::iterator itr = dstcks.begin();
842 for( ; itr != dstcks.end(); ++itr )
843 if( local_supported.count( *itr ) ) return *itr;
844 return std::string();
845 }
846
847 //--------------------------------------------------------------------------
848 // then check if destination is local
849 //--------------------------------------------------------------------------
850 if( destination.IsLocalFile() )
851 {
852 std::vector<std::string>::iterator itr = srccks.begin();
853 for( ; itr != srccks.end(); ++itr )
854 if( local_supported.count( *itr ) ) return *itr;
855 return std::string();
856 }
857
858 //--------------------------------------------------------------------------
859 // if both source and destination are remote look for a checksum that can
860 // satisfy both
861 //--------------------------------------------------------------------------
862 std::set<std::string> dst_supported( dstcks.begin(), dstcks.end() );
863 std::vector<std::string>::iterator itr = srccks.begin();
864 for( ; itr != srccks.end(); ++itr )
865 if( dst_supported.count( *itr ) ) return *itr;
866 return std::string();
867 }
868
869 //----------------------------------------------------------------------------
871 //----------------------------------------------------------------------------
872 void Utils::SplitChunks( std::vector<ChunkList> &listsvec,
873 const ChunkList &chunks,
874 const uint32_t maxcs,
875 const size_t maxc )
876 {
877 listsvec.clear();
878 if( !chunks.size() ) return;
879
880 listsvec.emplace_back();
881 ChunkList *c = &listsvec.back();
882 const size_t cs = chunks.size();
883 size_t idx = 0;
884 size_t nc = 0;
885 ChunkInfo tmpc;
886
887 c->reserve( cs );
888
889 while( idx < cs )
890 {
891 if( maxc && nc >= maxc )
892 {
893 listsvec.emplace_back();
894 c = &listsvec.back();
895 c->reserve( cs - idx );
896 nc = 0;
897 }
898
899 if( tmpc.length == 0 )
900 tmpc = chunks[idx];
901
902 if( maxcs && tmpc.length > maxcs )
903 {
904 c->emplace_back( tmpc.offset, maxcs, tmpc.buffer );
905 tmpc.offset += maxcs;
906 tmpc.length -= maxcs;
907 tmpc.buffer = static_cast<char*>( tmpc.buffer ) + maxcs;
908 }
909 else
910 {
911 c->emplace_back( tmpc.offset, tmpc.length, tmpc.buffer );
912 tmpc.length = 0;
913 ++idx;
914 }
915 ++nc;
916 }
917 }
918}
kXR_unt16 options
Definition XProtocol.hh:481
@ kXR_open_wrto
Definition XProtocol.hh:469
@ kXR_open_updt
Definition XProtocol.hh:457
@ kXR_new
Definition XProtocol.hh:455
struct ClientOpenRequest open
Definition XProtocol.hh:858
#define opendir(a)
Definition XrdPosix.hh:73
#define closedir(a)
Definition XrdPosix.hh:45
#define readdir(a)
Definition XrdPosix.hh:81
struct myOpts opts
int Set(const char *csName)
Definition XrdCksData.hh:81
int Get(char *Buff, int Blen)
Definition XrdCksData.hh:69
Binary blob representation.
void FromString(const std::string str)
Fill the buffer from a string.
const char * GetBuffer(uint32_t offset=0) const
Get the message buffer.
std::string ToString() const
Convert the buffer to a string.
Manage the checksum calc objects.
bool Calculate(XrdCksData &result, const std::string &algName, const std::string &filePath)
Calculate a checksum of for a given file.
static CheckSumManager * GetCheckSumManager()
Get checksum manager.
static Log * GetLog()
Get default log.
static Env * GetEnv()
Get default client environment.
bool GetString(const std::string &key, std::string &value)
Definition XrdClEnv.cc:31
bool GetInt(const std::string &key, int &value)
Definition XrdClEnv.cc:89
Send file/filesystem queries to an XRootD cluster.
XRootDStatus Query(QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
Handle diagnostics.
Definition XrdClLog.hh:101
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition XrdClLog.cc:231
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
Definition XrdClLog.cc:299
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition XrdClLog.cc:282
The message representation used throughout the system.
A key-value pair map storing both keys and values as strings.
PropertyMap::const_iterator end() const
Get the end iterator.
PropertyMap::const_iterator begin() const
Get the begin iterator.
Singleton access to URL to virtual redirector mapping.
static RedirectorRegistry & Instance()
Returns reference to the single instance.
VirtualRedirector * Get(const URL &url) const
Get a virtual redirector associated with the given URL.
URL representation.
Definition XrdClURL.hh:31
const std::string & GetPath() const
Get the path.
Definition XrdClURL.hh:212
bool IsMetalink() const
Is it a URL to a metalink.
Definition XrdClURL.cc:451
std::map< std::string, std::string > ParamsMap
Definition XrdClURL.hh:33
const std::string & GetHostName() const
Get the name of the target host.
Definition XrdClURL.hh:165
bool IsLocalFile() const
Definition XrdClURL.cc:460
const ParamsMap & GetParams() const
Get the URL params.
Definition XrdClURL.hh:239
const std::string & GetProtocol() const
Get the protocol.
Definition XrdClURL.hh:113
int GetPort() const
Get the target port.
Definition XrdClURL.hh:183
static std::string TimeToString(time_t timestamp)
Convert timestamp to a string.
static XRootDStatus CheckTPCLite(const std::string &server, uint16_t timeout=0)
static void LogHostAddresses(Log *log, uint64_t type, const std::string &hostId, std::vector< XrdNetAddr > &addresses)
Log all the addresses on the list.
static std::string NormalizeChecksum(const std::string &name, const std::string &checksum)
Normalize checksum.
static Status ProcessConfig(std::map< std::string, std::string > &config, const std::string &file)
Process a config file and return key-value pairs.
static Status ProcessConfigDir(std::map< std::string, std::string > &config, const std::string &dir)
Process a config directory and return key-value pairs.
static std::string FQDNToCC(const std::string &fqdn)
Convert the fully qualified host name to country code.
static std::string InferChecksumType(const XrdCl::URL &source, const XrdCl::URL &destination, bool zip=false)
Automatically infer the right checksum type.
static void LogPropertyList(Log *log, uint64_t topic, const char *format, const PropertyList &list)
Log property list.
static std::string Char2Hex(uint8_t *array, uint16_t size)
Print a char array as hex.
static void splitString(Container &result, const std::string &input, const std::string &delimiter)
Split a string.
Definition XrdClUtils.hh:56
static Status GetHostAddresses(std::vector< XrdNetAddr > &addresses, const URL &url, AddressType type)
Resolve IP addresses.
static uint64_t GetElapsedMicroSecs(timeval start, timeval end)
Get the elapsed microseconds between two timevals.
static std::vector< std::string > GetSupportedCheckSums(const XrdCl::URL &url)
Get supported checksum types for given URL.
static AddressType String2AddressType(const std::string &addressType)
Interpret a string as address type, default to IPAll.
static int GetIntParameter(const URL &url, const std::string &name, int defaultVal)
Get a parameter either from the environment or URL.
Definition XrdClUtils.cc:80
static Status GetDirectoryEntries(std::vector< std::string > &entries, const std::string &path)
Get directory entries.
static XRootDStatus GetLocalCheckSum(std::string &checkSum, const std::string &checkSumType, const std::string &path)
Get a checksum from local file.
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
static void Trim(std::string &str)
Trim a string.
static bool CheckEC(const Message *req, const URL &url)
Check if this client can support given EC redirect.
static XRootDStatus GetRemoteCheckSum(std::string &checkSum, const std::string &checkSumType, const URL &url)
Get a checksum from a remote xrootd server.
static std::string GetStringParameter(const URL &url, const std::string &name, const std::string &defaultVal)
Get a parameter either from the environment or URL.
static XRootDStatus CheckTPC(const std::string &server, uint16_t timeout=0)
Check if peer supports tpc.
AddressType
Address type.
Definition XrdClUtils.hh:87
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
An interface for metadata redirectors.
virtual std::vector< std::string > GetSupportedCheckSums() const =0
const std::string & GetErrorMessage() const
Get error message.
void SetErrorMessage(const std::string &message)
Set the error message.
std::string ToStr() const
Convert to string.
bool isMapped() const
bool isIPType(IPType ipType) const
static const char * GetAddrs(const char *hSpec, XrdNetAddr *aListP[], int &aListN, AddrOpts opts=allIPMap, int pNum=PortInSpec)
const uint16_t suPartial
const uint16_t errInvalidAddr
const uint16_t stFatal
Fatal error, it's still an error.
const uint16_t stError
An error occurred that could potentially be retried.
const uint16_t errInternal
Internal error.
const uint16_t stOK
Everything went OK.
const int DefaultIPNoShuffle
const uint16_t errConfig
System misconfigured.
const uint16_t errOSError
const uint64_t UtilityMsg
const uint16_t errInvalidResponse
std::vector< ChunkInfo > ChunkList
List of chunks.
const uint16_t errNotSupported
const int DefaultPreferIPv4
const uint16_t errCheckSumError
const int DefaultZipMtlnCksum
Describe a data chunk for vector read.
void * buffer
length of the chunk
uint32_t length
offset in the file
@ Config
Query server configuration.
@ Checksum
Query file checksum.
Procedure execution status.
uint16_t status
Status of the execution.
bool IsOK() const
We're fine.
std::string ToString() const
Create a string representation.