Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * basebackup.c
4 : * code for taking a base backup and streaming it to a standby
5 : *
6 : * Portions Copyright (c) 2010-2017, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/replication/basebackup.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres.h"
14 :
15 : #include <sys/stat.h>
16 : #include <unistd.h>
17 : #include <time.h>
18 :
19 : #include "access/xlog_internal.h" /* for pg_start/stop_backup */
20 : #include "catalog/catalog.h"
21 : #include "catalog/pg_type.h"
22 : #include "lib/stringinfo.h"
23 : #include "libpq/libpq.h"
24 : #include "libpq/pqformat.h"
25 : #include "miscadmin.h"
26 : #include "nodes/pg_list.h"
27 : #include "pgtar.h"
28 : #include "pgstat.h"
29 : #include "postmaster/syslogger.h"
30 : #include "replication/basebackup.h"
31 : #include "replication/walsender.h"
32 : #include "replication/walsender_private.h"
33 : #include "storage/dsm_impl.h"
34 : #include "storage/fd.h"
35 : #include "storage/ipc.h"
36 : #include "utils/builtins.h"
37 : #include "utils/elog.h"
38 : #include "utils/ps_status.h"
39 : #include "utils/timestamp.h"
40 :
41 :
42 : typedef struct
43 : {
44 : const char *label;
45 : bool progress;
46 : bool fastcheckpoint;
47 : bool nowait;
48 : bool includewal;
49 : uint32 maxrate;
50 : bool sendtblspcmapfile;
51 : } basebackup_options;
52 :
53 :
54 : static int64 sendDir(char *path, int basepathlen, bool sizeonly,
55 : List *tablespaces, bool sendtblspclinks);
56 : static bool sendFile(char *readfilename, char *tarfilename,
57 : struct stat *statbuf, bool missing_ok);
58 : static void sendFileWithContent(const char *filename, const char *content);
59 : static int64 _tarWriteHeader(const char *filename, const char *linktarget,
60 : struct stat *statbuf, bool sizeonly);
61 : static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
62 : bool sizeonly);
63 : static void send_int8_string(StringInfoData *buf, int64 intval);
64 : static void SendBackupHeader(List *tablespaces);
65 : static void base_backup_cleanup(int code, Datum arg);
66 : static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir);
67 : static void parse_basebackup_options(List *options, basebackup_options *opt);
68 : static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
69 : static int compareWalFileNames(const void *a, const void *b);
70 : static void throttle(size_t increment);
71 :
72 : /* Was the backup currently in-progress initiated in recovery mode? */
73 : static bool backup_started_in_recovery = false;
74 :
75 : /* Relative path of temporary statistics directory */
76 : static char *statrelpath = NULL;
77 :
78 : /*
79 : * Size of each block sent into the tar stream for larger files.
80 : */
81 : #define TAR_SEND_SIZE 32768
82 :
83 : /*
84 : * How frequently to throttle, as a fraction of the specified rate-second.
85 : */
86 : #define THROTTLING_FREQUENCY 8
87 :
88 : /* The actual number of bytes, transfer of which may cause sleep. */
89 : static uint64 throttling_sample;
90 :
91 : /* Amount of data already transferred but not yet throttled. */
92 : static int64 throttling_counter;
93 :
94 : /* The minimum time required to transfer throttling_sample bytes. */
95 : static TimeOffset elapsed_min_unit;
96 :
97 : /* The last check of the transfer rate. */
98 : static TimestampTz throttled_last;
99 :
100 : /*
101 : * The contents of these directories are removed or recreated during server
102 : * start so they are not included in backups. The directories themselves are
103 : * kept and included as empty to preserve access permissions.
104 : */
105 : static const char *excludeDirContents[] =
106 : {
107 : /*
108 : * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
109 : * when stats_temp_directory is set because PGSS_TEXT_FILE is always
110 : * created there.
111 : */
112 : PG_STAT_TMP_DIR,
113 :
114 : /*
115 : * It is generally not useful to backup the contents of this directory
116 : * even if the intention is to restore to another master. See backup.sgml
117 : * for a more detailed description.
118 : */
119 : "pg_replslot",
120 :
121 : /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
122 : PG_DYNSHMEM_DIR,
123 :
124 : /* Contents removed on startup, see AsyncShmemInit(). */
125 : "pg_notify",
126 :
127 : /*
128 : * Old contents are loaded for possible debugging but are not required for
129 : * normal operation, see OldSerXidInit().
130 : */
131 : "pg_serial",
132 :
133 : /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
134 : "pg_snapshots",
135 :
136 : /* Contents zeroed on startup, see StartupSUBTRANS(). */
137 : "pg_subtrans",
138 :
139 : /* end of list */
140 : NULL
141 : };
142 :
143 : /*
144 : * List of files excluded from backups.
145 : */
146 : static const char *excludeFiles[] =
147 : {
148 : /* Skip auto conf temporary file. */
149 : PG_AUTOCONF_FILENAME ".tmp",
150 :
151 : /* Skip current log file temporary file */
152 : LOG_METAINFO_DATAFILE_TMP,
153 :
154 : /*
155 : * If there's a backup_label or tablespace_map file, it belongs to a
156 : * backup started by the user with pg_start_backup(). It is *not* correct
157 : * for this backup. Our backup_label/tablespace_map is injected into the
158 : * tar separately.
159 : */
160 : BACKUP_LABEL_FILE,
161 : TABLESPACE_MAP,
162 :
163 : "postmaster.pid",
164 : "postmaster.opts",
165 :
166 : /* end of list */
167 : NULL
168 : };
169 :
170 : /*
171 : * Called when ERROR or FATAL happens in perform_base_backup() after
172 : * we have started the backup - make sure we end it!
173 : */
174 : static void
175 0 : base_backup_cleanup(int code, Datum arg)
176 : {
177 0 : do_pg_abort_backup();
178 0 : }
179 :
180 : /*
181 : * Actually do a base backup for the specified tablespaces.
182 : *
183 : * This is split out mainly to avoid complaints about "variable might be
184 : * clobbered by longjmp" from stupider versions of gcc.
185 : */
186 : static void
187 0 : perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
188 : {
189 : XLogRecPtr startptr;
190 : TimeLineID starttli;
191 : XLogRecPtr endptr;
192 : TimeLineID endtli;
193 : StringInfo labelfile;
194 0 : StringInfo tblspc_map_file = NULL;
195 : int datadirpathlen;
196 0 : List *tablespaces = NIL;
197 :
198 0 : datadirpathlen = strlen(DataDir);
199 :
200 0 : backup_started_in_recovery = RecoveryInProgress();
201 :
202 0 : labelfile = makeStringInfo();
203 0 : tblspc_map_file = makeStringInfo();
204 :
205 0 : startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
206 : labelfile, tblspcdir, &tablespaces,
207 : tblspc_map_file,
208 0 : opt->progress, opt->sendtblspcmapfile);
209 :
210 : /*
211 : * Once do_pg_start_backup has been called, ensure that any failure causes
212 : * us to abort the backup so we don't "leak" a backup counter. For this
213 : * reason, *all* functionality between do_pg_start_backup() and
214 : * do_pg_stop_backup() should be inside the error cleanup block!
215 : */
216 :
217 0 : PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0);
218 : {
219 : ListCell *lc;
220 : tablespaceinfo *ti;
221 :
222 0 : SendXlogRecPtrResult(startptr, starttli);
223 :
224 : /*
225 : * Calculate the relative path of temporary statistics directory in
226 : * order to skip the files which are located in that directory later.
227 : */
228 0 : if (is_absolute_path(pgstat_stat_directory) &&
229 0 : strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
230 0 : statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
231 0 : else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
232 0 : statrelpath = psprintf("./%s", pgstat_stat_directory);
233 : else
234 0 : statrelpath = pgstat_stat_directory;
235 :
236 : /* Add a node for the base directory at the end */
237 0 : ti = palloc0(sizeof(tablespaceinfo));
238 0 : ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1;
239 0 : tablespaces = lappend(tablespaces, ti);
240 :
241 : /* Send tablespace header */
242 0 : SendBackupHeader(tablespaces);
243 :
244 : /* Setup and activate network throttling, if client requested it */
245 0 : if (opt->maxrate > 0)
246 : {
247 0 : throttling_sample =
248 0 : (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
249 :
250 : /*
251 : * The minimum amount of time for throttling_sample bytes to be
252 : * transferred.
253 : */
254 0 : elapsed_min_unit = USECS_PER_SEC / THROTTLING_FREQUENCY;
255 :
256 : /* Enable throttling. */
257 0 : throttling_counter = 0;
258 :
259 : /* The 'real data' starts now (header was ignored). */
260 0 : throttled_last = GetCurrentTimestamp();
261 : }
262 : else
263 : {
264 : /* Disable throttling. */
265 0 : throttling_counter = -1;
266 : }
267 :
268 : /* Send off our tablespaces one by one */
269 0 : foreach(lc, tablespaces)
270 : {
271 0 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
272 : StringInfoData buf;
273 :
274 : /* Send CopyOutResponse message */
275 0 : pq_beginmessage(&buf, 'H');
276 0 : pq_sendbyte(&buf, 0); /* overall format */
277 0 : pq_sendint(&buf, 0, 2); /* natts */
278 0 : pq_endmessage(&buf);
279 :
280 0 : if (ti->path == NULL)
281 : {
282 : struct stat statbuf;
283 :
284 : /* In the main tar, include the backup_label first... */
285 0 : sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data);
286 :
287 : /*
288 : * Send tablespace_map file if required and then the bulk of
289 : * the files.
290 : */
291 0 : if (tblspc_map_file && opt->sendtblspcmapfile)
292 : {
293 0 : sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data);
294 0 : sendDir(".", 1, false, tablespaces, false);
295 : }
296 : else
297 0 : sendDir(".", 1, false, tablespaces, true);
298 :
299 : /* ... and pg_control after everything else. */
300 0 : if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
301 0 : ereport(ERROR,
302 : (errcode_for_file_access(),
303 : errmsg("could not stat control file \"%s\": %m",
304 : XLOG_CONTROL_FILE)));
305 0 : sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false);
306 : }
307 : else
308 0 : sendTablespace(ti->path, false);
309 :
310 : /*
311 : * If we're including WAL, and this is the main data directory we
312 : * don't terminate the tar stream here. Instead, we will append
313 : * the xlog files below and terminate it then. This is safe since
314 : * the main data directory is always sent *last*.
315 : */
316 0 : if (opt->includewal && ti->path == NULL)
317 : {
318 0 : Assert(lnext(lc) == NULL);
319 : }
320 : else
321 0 : pq_putemptymessage('c'); /* CopyDone */
322 : }
323 : }
324 0 : PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0);
325 :
326 0 : endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
327 :
328 0 : if (opt->includewal)
329 : {
330 : /*
331 : * We've left the last tar file "open", so we can now append the
332 : * required WAL files to it.
333 : */
334 : char pathbuf[MAXPGPATH];
335 : XLogSegNo segno;
336 : XLogSegNo startsegno;
337 : XLogSegNo endsegno;
338 : struct stat statbuf;
339 0 : List *historyFileList = NIL;
340 0 : List *walFileList = NIL;
341 : char **walFiles;
342 : int nWalFiles;
343 : char firstoff[MAXFNAMELEN];
344 : char lastoff[MAXFNAMELEN];
345 : DIR *dir;
346 : struct dirent *de;
347 : int i;
348 : ListCell *lc;
349 : TimeLineID tli;
350 :
351 : /*
352 : * I'd rather not worry about timelines here, so scan pg_wal and
353 : * include all WAL files in the range between 'startptr' and 'endptr',
354 : * regardless of the timeline the file is stamped with. If there are
355 : * some spurious WAL files belonging to timelines that don't belong in
356 : * this server's history, they will be included too. Normally there
357 : * shouldn't be such files, but if there are, there's little harm in
358 : * including them.
359 : */
360 0 : XLByteToSeg(startptr, startsegno);
361 0 : XLogFileName(firstoff, ThisTimeLineID, startsegno);
362 0 : XLByteToPrevSeg(endptr, endsegno);
363 0 : XLogFileName(lastoff, ThisTimeLineID, endsegno);
364 :
365 0 : dir = AllocateDir("pg_wal");
366 0 : if (!dir)
367 0 : ereport(ERROR,
368 : (errmsg("could not open directory \"%s\": %m", "pg_wal")));
369 0 : while ((de = ReadDir(dir, "pg_wal")) != NULL)
370 : {
371 : /* Does it look like a WAL segment, and is it in the range? */
372 0 : if (IsXLogFileName(de->d_name) &&
373 0 : strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
374 0 : strcmp(de->d_name + 8, lastoff + 8) <= 0)
375 : {
376 0 : walFileList = lappend(walFileList, pstrdup(de->d_name));
377 : }
378 : /* Does it look like a timeline history file? */
379 0 : else if (IsTLHistoryFileName(de->d_name))
380 : {
381 0 : historyFileList = lappend(historyFileList, pstrdup(de->d_name));
382 : }
383 : }
384 0 : FreeDir(dir);
385 :
386 : /*
387 : * Before we go any further, check that none of the WAL segments we
388 : * need were removed.
389 : */
390 0 : CheckXLogRemoved(startsegno, ThisTimeLineID);
391 :
392 : /*
393 : * Put the WAL filenames into an array, and sort. We send the files in
394 : * order from oldest to newest, to reduce the chance that a file is
395 : * recycled before we get a chance to send it over.
396 : */
397 0 : nWalFiles = list_length(walFileList);
398 0 : walFiles = palloc(nWalFiles * sizeof(char *));
399 0 : i = 0;
400 0 : foreach(lc, walFileList)
401 : {
402 0 : walFiles[i++] = lfirst(lc);
403 : }
404 0 : qsort(walFiles, nWalFiles, sizeof(char *), compareWalFileNames);
405 :
406 : /*
407 : * There must be at least one xlog file in the pg_wal directory, since
408 : * we are doing backup-including-xlog.
409 : */
410 0 : if (nWalFiles < 1)
411 0 : ereport(ERROR,
412 : (errmsg("could not find any WAL files")));
413 :
414 : /*
415 : * Sanity check: the first and last segment should cover startptr and
416 : * endptr, with no gaps in between.
417 : */
418 0 : XLogFromFileName(walFiles[0], &tli, &segno);
419 0 : if (segno != startsegno)
420 : {
421 : char startfname[MAXFNAMELEN];
422 :
423 0 : XLogFileName(startfname, ThisTimeLineID, startsegno);
424 0 : ereport(ERROR,
425 : (errmsg("could not find WAL file \"%s\"", startfname)));
426 : }
427 0 : for (i = 0; i < nWalFiles; i++)
428 : {
429 0 : XLogSegNo currsegno = segno;
430 0 : XLogSegNo nextsegno = segno + 1;
431 :
432 0 : XLogFromFileName(walFiles[i], &tli, &segno);
433 0 : if (!(nextsegno == segno || currsegno == segno))
434 : {
435 : char nextfname[MAXFNAMELEN];
436 :
437 0 : XLogFileName(nextfname, ThisTimeLineID, nextsegno);
438 0 : ereport(ERROR,
439 : (errmsg("could not find WAL file \"%s\"", nextfname)));
440 : }
441 : }
442 0 : if (segno != endsegno)
443 : {
444 : char endfname[MAXFNAMELEN];
445 :
446 0 : XLogFileName(endfname, ThisTimeLineID, endsegno);
447 0 : ereport(ERROR,
448 : (errmsg("could not find WAL file \"%s\"", endfname)));
449 : }
450 :
451 : /* Ok, we have everything we need. Send the WAL files. */
452 0 : for (i = 0; i < nWalFiles; i++)
453 : {
454 : FILE *fp;
455 : char buf[TAR_SEND_SIZE];
456 : size_t cnt;
457 0 : pgoff_t len = 0;
458 :
459 0 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
460 0 : XLogFromFileName(walFiles[i], &tli, &segno);
461 :
462 0 : fp = AllocateFile(pathbuf, "rb");
463 0 : if (fp == NULL)
464 : {
465 : /*
466 : * Most likely reason for this is that the file was already
467 : * removed by a checkpoint, so check for that to get a better
468 : * error message.
469 : */
470 0 : CheckXLogRemoved(segno, tli);
471 :
472 0 : ereport(ERROR,
473 : (errcode_for_file_access(),
474 : errmsg("could not open file \"%s\": %m", pathbuf)));
475 : }
476 :
477 0 : if (fstat(fileno(fp), &statbuf) != 0)
478 0 : ereport(ERROR,
479 : (errcode_for_file_access(),
480 : errmsg("could not stat file \"%s\": %m",
481 : pathbuf)));
482 0 : if (statbuf.st_size != XLogSegSize)
483 : {
484 0 : CheckXLogRemoved(segno, tli);
485 0 : ereport(ERROR,
486 : (errcode_for_file_access(),
487 : errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
488 : }
489 :
490 : /* send the WAL file itself */
491 0 : _tarWriteHeader(pathbuf, NULL, &statbuf, false);
492 :
493 0 : while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0)
494 : {
495 0 : CheckXLogRemoved(segno, tli);
496 : /* Send the chunk as a CopyData message */
497 0 : if (pq_putmessage('d', buf, cnt))
498 0 : ereport(ERROR,
499 : (errmsg("base backup could not send data, aborting backup")));
500 :
501 0 : len += cnt;
502 0 : throttle(cnt);
503 :
504 0 : if (len == XLogSegSize)
505 0 : break;
506 : }
507 :
508 0 : if (len != XLogSegSize)
509 : {
510 0 : CheckXLogRemoved(segno, tli);
511 0 : ereport(ERROR,
512 : (errcode_for_file_access(),
513 : errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
514 : }
515 :
516 : /* XLogSegSize is a multiple of 512, so no need for padding */
517 :
518 0 : FreeFile(fp);
519 :
520 : /*
521 : * Mark file as archived, otherwise files can get archived again
522 : * after promotion of a new node. This is in line with
523 : * walreceiver.c always doing an XLogArchiveForceDone() after a
524 : * complete segment.
525 : */
526 0 : StatusFilePath(pathbuf, walFiles[i], ".done");
527 0 : sendFileWithContent(pathbuf, "");
528 : }
529 :
530 : /*
531 : * Send timeline history files too. Only the latest timeline history
532 : * file is required for recovery, and even that only if there happens
533 : * to be a timeline switch in the first WAL segment that contains the
534 : * checkpoint record, or if we're taking a base backup from a standby
535 : * server and the target timeline changes while the backup is taken.
536 : * But they are small and highly useful for debugging purposes, so
537 : * better include them all, always.
538 : */
539 0 : foreach(lc, historyFileList)
540 : {
541 0 : char *fname = lfirst(lc);
542 :
543 0 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
544 :
545 0 : if (lstat(pathbuf, &statbuf) != 0)
546 0 : ereport(ERROR,
547 : (errcode_for_file_access(),
548 : errmsg("could not stat file \"%s\": %m", pathbuf)));
549 :
550 0 : sendFile(pathbuf, pathbuf, &statbuf, false);
551 :
552 : /* unconditionally mark file as archived */
553 0 : StatusFilePath(pathbuf, fname, ".done");
554 0 : sendFileWithContent(pathbuf, "");
555 : }
556 :
557 : /* Send CopyDone message for the last tar file */
558 0 : pq_putemptymessage('c');
559 : }
560 0 : SendXlogRecPtrResult(endptr, endtli);
561 0 : }
562 :
563 : /*
564 : * qsort comparison function, to compare log/seg portion of WAL segment
565 : * filenames, ignoring the timeline portion.
566 : */
567 : static int
568 0 : compareWalFileNames(const void *a, const void *b)
569 : {
570 0 : char *fna = *((char **) a);
571 0 : char *fnb = *((char **) b);
572 :
573 0 : return strcmp(fna + 8, fnb + 8);
574 : }
575 :
576 : /*
577 : * Parse the base backup options passed down by the parser
578 : */
579 : static void
580 0 : parse_basebackup_options(List *options, basebackup_options *opt)
581 : {
582 : ListCell *lopt;
583 0 : bool o_label = false;
584 0 : bool o_progress = false;
585 0 : bool o_fast = false;
586 0 : bool o_nowait = false;
587 0 : bool o_wal = false;
588 0 : bool o_maxrate = false;
589 0 : bool o_tablespace_map = false;
590 :
591 0 : MemSet(opt, 0, sizeof(*opt));
592 0 : foreach(lopt, options)
593 : {
594 0 : DefElem *defel = (DefElem *) lfirst(lopt);
595 :
596 0 : if (strcmp(defel->defname, "label") == 0)
597 : {
598 0 : if (o_label)
599 0 : ereport(ERROR,
600 : (errcode(ERRCODE_SYNTAX_ERROR),
601 : errmsg("duplicate option \"%s\"", defel->defname)));
602 0 : opt->label = strVal(defel->arg);
603 0 : o_label = true;
604 : }
605 0 : else if (strcmp(defel->defname, "progress") == 0)
606 : {
607 0 : if (o_progress)
608 0 : ereport(ERROR,
609 : (errcode(ERRCODE_SYNTAX_ERROR),
610 : errmsg("duplicate option \"%s\"", defel->defname)));
611 0 : opt->progress = true;
612 0 : o_progress = true;
613 : }
614 0 : else if (strcmp(defel->defname, "fast") == 0)
615 : {
616 0 : if (o_fast)
617 0 : ereport(ERROR,
618 : (errcode(ERRCODE_SYNTAX_ERROR),
619 : errmsg("duplicate option \"%s\"", defel->defname)));
620 0 : opt->fastcheckpoint = true;
621 0 : o_fast = true;
622 : }
623 0 : else if (strcmp(defel->defname, "nowait") == 0)
624 : {
625 0 : if (o_nowait)
626 0 : ereport(ERROR,
627 : (errcode(ERRCODE_SYNTAX_ERROR),
628 : errmsg("duplicate option \"%s\"", defel->defname)));
629 0 : opt->nowait = true;
630 0 : o_nowait = true;
631 : }
632 0 : else if (strcmp(defel->defname, "wal") == 0)
633 : {
634 0 : if (o_wal)
635 0 : ereport(ERROR,
636 : (errcode(ERRCODE_SYNTAX_ERROR),
637 : errmsg("duplicate option \"%s\"", defel->defname)));
638 0 : opt->includewal = true;
639 0 : o_wal = true;
640 : }
641 0 : else if (strcmp(defel->defname, "max_rate") == 0)
642 : {
643 : long maxrate;
644 :
645 0 : if (o_maxrate)
646 0 : ereport(ERROR,
647 : (errcode(ERRCODE_SYNTAX_ERROR),
648 : errmsg("duplicate option \"%s\"", defel->defname)));
649 :
650 0 : maxrate = intVal(defel->arg);
651 0 : if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
652 0 : ereport(ERROR,
653 : (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
654 : errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
655 : (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
656 :
657 0 : opt->maxrate = (uint32) maxrate;
658 0 : o_maxrate = true;
659 : }
660 0 : else if (strcmp(defel->defname, "tablespace_map") == 0)
661 : {
662 0 : if (o_tablespace_map)
663 0 : ereport(ERROR,
664 : (errcode(ERRCODE_SYNTAX_ERROR),
665 : errmsg("duplicate option \"%s\"", defel->defname)));
666 0 : opt->sendtblspcmapfile = true;
667 0 : o_tablespace_map = true;
668 : }
669 : else
670 0 : elog(ERROR, "option \"%s\" not recognized",
671 : defel->defname);
672 : }
673 0 : if (opt->label == NULL)
674 0 : opt->label = "base backup";
675 0 : }
676 :
677 :
678 : /*
679 : * SendBaseBackup() - send a complete base backup.
680 : *
681 : * The function will put the system into backup mode like pg_start_backup()
682 : * does, so that the backup is consistent even though we read directly from
683 : * the filesystem, bypassing the buffer cache.
684 : */
685 : void
686 0 : SendBaseBackup(BaseBackupCmd *cmd)
687 : {
688 : DIR *dir;
689 : basebackup_options opt;
690 :
691 0 : parse_basebackup_options(cmd->options, &opt);
692 :
693 0 : WalSndSetState(WALSNDSTATE_BACKUP);
694 :
695 0 : if (update_process_title)
696 : {
697 : char activitymsg[50];
698 :
699 0 : snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
700 : opt.label);
701 0 : set_ps_display(activitymsg, false);
702 : }
703 :
704 : /* Make sure we can open the directory with tablespaces in it */
705 0 : dir = AllocateDir("pg_tblspc");
706 0 : if (!dir)
707 0 : ereport(ERROR,
708 : (errmsg("could not open directory \"%s\": %m", "pg_tblspc")));
709 :
710 0 : perform_base_backup(&opt, dir);
711 :
712 0 : FreeDir(dir);
713 0 : }
714 :
715 : static void
716 0 : send_int8_string(StringInfoData *buf, int64 intval)
717 : {
718 : char is[32];
719 :
720 0 : sprintf(is, INT64_FORMAT, intval);
721 0 : pq_sendint(buf, strlen(is), 4);
722 0 : pq_sendbytes(buf, is, strlen(is));
723 0 : }
724 :
725 : static void
726 0 : SendBackupHeader(List *tablespaces)
727 : {
728 : StringInfoData buf;
729 : ListCell *lc;
730 :
731 : /* Construct and send the directory information */
732 0 : pq_beginmessage(&buf, 'T'); /* RowDescription */
733 0 : pq_sendint(&buf, 3, 2); /* 3 fields */
734 :
735 : /* First field - spcoid */
736 0 : pq_sendstring(&buf, "spcoid");
737 0 : pq_sendint(&buf, 0, 4); /* table oid */
738 0 : pq_sendint(&buf, 0, 2); /* attnum */
739 0 : pq_sendint(&buf, OIDOID, 4); /* type oid */
740 0 : pq_sendint(&buf, 4, 2); /* typlen */
741 0 : pq_sendint(&buf, 0, 4); /* typmod */
742 0 : pq_sendint(&buf, 0, 2); /* format code */
743 :
744 : /* Second field - spcpath */
745 0 : pq_sendstring(&buf, "spclocation");
746 0 : pq_sendint(&buf, 0, 4);
747 0 : pq_sendint(&buf, 0, 2);
748 0 : pq_sendint(&buf, TEXTOID, 4);
749 0 : pq_sendint(&buf, -1, 2);
750 0 : pq_sendint(&buf, 0, 4);
751 0 : pq_sendint(&buf, 0, 2);
752 :
753 : /* Third field - size */
754 0 : pq_sendstring(&buf, "size");
755 0 : pq_sendint(&buf, 0, 4);
756 0 : pq_sendint(&buf, 0, 2);
757 0 : pq_sendint(&buf, INT8OID, 4);
758 0 : pq_sendint(&buf, 8, 2);
759 0 : pq_sendint(&buf, 0, 4);
760 0 : pq_sendint(&buf, 0, 2);
761 0 : pq_endmessage(&buf);
762 :
763 0 : foreach(lc, tablespaces)
764 : {
765 0 : tablespaceinfo *ti = lfirst(lc);
766 :
767 : /* Send one datarow message */
768 0 : pq_beginmessage(&buf, 'D');
769 0 : pq_sendint(&buf, 3, 2); /* number of columns */
770 0 : if (ti->path == NULL)
771 : {
772 0 : pq_sendint(&buf, -1, 4); /* Length = -1 ==> NULL */
773 0 : pq_sendint(&buf, -1, 4);
774 : }
775 : else
776 : {
777 : Size len;
778 :
779 0 : len = strlen(ti->oid);
780 0 : pq_sendint(&buf, len, 4);
781 0 : pq_sendbytes(&buf, ti->oid, len);
782 :
783 0 : len = strlen(ti->path);
784 0 : pq_sendint(&buf, len, 4);
785 0 : pq_sendbytes(&buf, ti->path, len);
786 : }
787 0 : if (ti->size >= 0)
788 0 : send_int8_string(&buf, ti->size / 1024);
789 : else
790 0 : pq_sendint(&buf, -1, 4); /* NULL */
791 :
792 0 : pq_endmessage(&buf);
793 : }
794 :
795 : /* Send a CommandComplete message */
796 0 : pq_puttextmessage('C', "SELECT");
797 0 : }
798 :
799 : /*
800 : * Send a single resultset containing just a single
801 : * XLogRecPtr record (in text format)
802 : */
803 : static void
804 0 : SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
805 : {
806 : StringInfoData buf;
807 : char str[MAXFNAMELEN];
808 : Size len;
809 :
810 0 : pq_beginmessage(&buf, 'T'); /* RowDescription */
811 0 : pq_sendint(&buf, 2, 2); /* 2 fields */
812 :
813 : /* Field headers */
814 0 : pq_sendstring(&buf, "recptr");
815 0 : pq_sendint(&buf, 0, 4); /* table oid */
816 0 : pq_sendint(&buf, 0, 2); /* attnum */
817 0 : pq_sendint(&buf, TEXTOID, 4); /* type oid */
818 0 : pq_sendint(&buf, -1, 2);
819 0 : pq_sendint(&buf, 0, 4);
820 0 : pq_sendint(&buf, 0, 2);
821 :
822 0 : pq_sendstring(&buf, "tli");
823 0 : pq_sendint(&buf, 0, 4); /* table oid */
824 0 : pq_sendint(&buf, 0, 2); /* attnum */
825 :
826 : /*
827 : * int8 may seem like a surprising data type for this, but in theory int4
828 : * would not be wide enough for this, as TimeLineID is unsigned.
829 : */
830 0 : pq_sendint(&buf, INT8OID, 4); /* type oid */
831 0 : pq_sendint(&buf, -1, 2);
832 0 : pq_sendint(&buf, 0, 4);
833 0 : pq_sendint(&buf, 0, 2);
834 0 : pq_endmessage(&buf);
835 :
836 : /* Data row */
837 0 : pq_beginmessage(&buf, 'D');
838 0 : pq_sendint(&buf, 2, 2); /* number of columns */
839 :
840 0 : len = snprintf(str, sizeof(str),
841 0 : "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
842 0 : pq_sendint(&buf, len, 4);
843 0 : pq_sendbytes(&buf, str, len);
844 :
845 0 : len = snprintf(str, sizeof(str), "%u", tli);
846 0 : pq_sendint(&buf, len, 4);
847 0 : pq_sendbytes(&buf, str, len);
848 :
849 0 : pq_endmessage(&buf);
850 :
851 : /* Send a CommandComplete message */
852 0 : pq_puttextmessage('C', "SELECT");
853 0 : }
854 :
855 : /*
856 : * Inject a file with given name and content in the output tar stream.
857 : */
858 : static void
859 0 : sendFileWithContent(const char *filename, const char *content)
860 : {
861 : struct stat statbuf;
862 : int pad,
863 : len;
864 :
865 0 : len = strlen(content);
866 :
867 : /*
868 : * Construct a stat struct for the backup_label file we're injecting in
869 : * the tar.
870 : */
871 : /* Windows doesn't have the concept of uid and gid */
872 : #ifdef WIN32
873 : statbuf.st_uid = 0;
874 : statbuf.st_gid = 0;
875 : #else
876 0 : statbuf.st_uid = geteuid();
877 0 : statbuf.st_gid = getegid();
878 : #endif
879 0 : statbuf.st_mtime = time(NULL);
880 0 : statbuf.st_mode = S_IRUSR | S_IWUSR;
881 0 : statbuf.st_size = len;
882 :
883 0 : _tarWriteHeader(filename, NULL, &statbuf, false);
884 : /* Send the contents as a CopyData message */
885 0 : pq_putmessage('d', content, len);
886 :
887 : /* Pad to 512 byte boundary, per tar format requirements */
888 0 : pad = ((len + 511) & ~511) - len;
889 0 : if (pad > 0)
890 : {
891 : char buf[512];
892 :
893 0 : MemSet(buf, 0, pad);
894 0 : pq_putmessage('d', buf, pad);
895 : }
896 0 : }
897 :
898 : /*
899 : * Include the tablespace directory pointed to by 'path' in the output tar
900 : * stream. If 'sizeonly' is true, we just calculate a total length and return
901 : * it, without actually sending anything.
902 : *
903 : * Only used to send auxiliary tablespaces, not PGDATA.
904 : */
905 : int64
906 0 : sendTablespace(char *path, bool sizeonly)
907 : {
908 : int64 size;
909 : char pathbuf[MAXPGPATH];
910 : struct stat statbuf;
911 :
912 : /*
913 : * 'path' points to the tablespace location, but we only want to include
914 : * the version directory in it that belongs to us.
915 : */
916 0 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
917 : TABLESPACE_VERSION_DIRECTORY);
918 :
919 : /*
920 : * Store a directory entry in the tar file so we get the permissions
921 : * right.
922 : */
923 0 : if (lstat(pathbuf, &statbuf) != 0)
924 : {
925 0 : if (errno != ENOENT)
926 0 : ereport(ERROR,
927 : (errcode_for_file_access(),
928 : errmsg("could not stat file or directory \"%s\": %m",
929 : pathbuf)));
930 :
931 : /* If the tablespace went away while scanning, it's no error. */
932 0 : return 0;
933 : }
934 :
935 0 : size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
936 : sizeonly);
937 :
938 : /* Send all the files in the tablespace version directory */
939 0 : size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true);
940 :
941 0 : return size;
942 : }
943 :
944 : /*
945 : * Include all files from the given directory in the output tar stream. If
946 : * 'sizeonly' is true, we just calculate a total length and return it, without
947 : * actually sending anything.
948 : *
949 : * Omit any directory in the tablespaces list, to avoid backing up
950 : * tablespaces twice when they were created inside PGDATA.
951 : *
952 : * If sendtblspclinks is true, we need to include symlink
953 : * information in the tar file. If not, we can skip that
954 : * as it will be sent separately in the tablespace_map file.
955 : */
956 : static int64
957 0 : sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces,
958 : bool sendtblspclinks)
959 : {
960 : DIR *dir;
961 : struct dirent *de;
962 : char pathbuf[MAXPGPATH * 2];
963 : struct stat statbuf;
964 0 : int64 size = 0;
965 :
966 0 : dir = AllocateDir(path);
967 0 : while ((de = ReadDir(dir, path)) != NULL)
968 : {
969 : int excludeIdx;
970 : bool excludeFound;
971 :
972 : /* Skip special stuff */
973 0 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
974 0 : continue;
975 :
976 : /* Skip temporary files */
977 0 : if (strncmp(de->d_name,
978 : PG_TEMP_FILE_PREFIX,
979 : strlen(PG_TEMP_FILE_PREFIX)) == 0)
980 0 : continue;
981 :
982 : /*
983 : * Check if the postmaster has signaled us to exit, and abort with an
984 : * error in that case. The error handler further up will call
985 : * do_pg_abort_backup() for us. Also check that if the backup was
986 : * started while still in recovery, the server wasn't promoted.
987 : * dp_pg_stop_backup() will check that too, but it's better to stop
988 : * the backup early than continue to the end and fail there.
989 : */
990 0 : CHECK_FOR_INTERRUPTS();
991 0 : if (RecoveryInProgress() != backup_started_in_recovery)
992 0 : ereport(ERROR,
993 : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
994 : errmsg("the standby was promoted during online backup"),
995 : errhint("This means that the backup being taken is corrupt "
996 : "and should not be used. "
997 : "Try taking another online backup.")));
998 :
999 : /* Scan for files that should be excluded */
1000 0 : excludeFound = false;
1001 0 : for (excludeIdx = 0; excludeFiles[excludeIdx] != NULL; excludeIdx++)
1002 : {
1003 0 : if (strcmp(de->d_name, excludeFiles[excludeIdx]) == 0)
1004 : {
1005 0 : elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1006 0 : excludeFound = true;
1007 0 : break;
1008 : }
1009 : }
1010 :
1011 0 : if (excludeFound)
1012 0 : continue;
1013 :
1014 0 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1015 :
1016 : /* Skip pg_control here to back up it last */
1017 0 : if (strcmp(pathbuf, "./global/pg_control") == 0)
1018 0 : continue;
1019 :
1020 0 : if (lstat(pathbuf, &statbuf) != 0)
1021 : {
1022 0 : if (errno != ENOENT)
1023 0 : ereport(ERROR,
1024 : (errcode_for_file_access(),
1025 : errmsg("could not stat file or directory \"%s\": %m",
1026 : pathbuf)));
1027 :
1028 : /* If the file went away while scanning, it's not an error. */
1029 0 : continue;
1030 : }
1031 :
1032 : /* Scan for directories whose contents should be excluded */
1033 0 : excludeFound = false;
1034 0 : for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1035 : {
1036 0 : if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1037 : {
1038 0 : elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1039 0 : size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1040 0 : excludeFound = true;
1041 0 : break;
1042 : }
1043 : }
1044 :
1045 0 : if (excludeFound)
1046 0 : continue;
1047 :
1048 : /*
1049 : * Exclude contents of directory specified by statrelpath if not set
1050 : * to the default (pg_stat_tmp) which is caught in the loop above.
1051 : */
1052 0 : if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1053 : {
1054 0 : elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1055 0 : size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1056 0 : continue;
1057 : }
1058 :
1059 : /*
1060 : * We can skip pg_wal, the WAL segments need to be fetched from the
1061 : * WAL archive anyway. But include it as an empty directory anyway, so
1062 : * we get permissions right.
1063 : */
1064 0 : if (strcmp(pathbuf, "./pg_wal") == 0)
1065 : {
1066 : /* If pg_wal is a symlink, write it as a directory anyway */
1067 0 : size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1068 :
1069 : /*
1070 : * Also send archive_status directory (by hackishly reusing
1071 : * statbuf from above ...).
1072 : */
1073 0 : size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1074 : sizeonly);
1075 :
1076 0 : continue; /* don't recurse into pg_wal */
1077 : }
1078 :
1079 : /* Allow symbolic links in pg_tblspc only */
1080 0 : if (strcmp(path, "./pg_tblspc") == 0 &&
1081 : #ifndef WIN32
1082 0 : S_ISLNK(statbuf.st_mode)
1083 : #else
1084 : pgwin32_is_junction(pathbuf)
1085 : #endif
1086 : )
1087 0 : {
1088 : #if defined(HAVE_READLINK) || defined(WIN32)
1089 : char linkpath[MAXPGPATH];
1090 : int rllen;
1091 :
1092 0 : rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1093 0 : if (rllen < 0)
1094 0 : ereport(ERROR,
1095 : (errcode_for_file_access(),
1096 : errmsg("could not read symbolic link \"%s\": %m",
1097 : pathbuf)));
1098 0 : if (rllen >= sizeof(linkpath))
1099 0 : ereport(ERROR,
1100 : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1101 : errmsg("symbolic link \"%s\" target is too long",
1102 : pathbuf)));
1103 0 : linkpath[rllen] = '\0';
1104 :
1105 0 : size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1106 : &statbuf, sizeonly);
1107 : #else
1108 :
1109 : /*
1110 : * If the platform does not have symbolic links, it should not be
1111 : * possible to have tablespaces - clearly somebody else created
1112 : * them. Warn about it and ignore.
1113 : */
1114 : ereport(WARNING,
1115 : (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1116 : errmsg("tablespaces are not supported on this platform")));
1117 : continue;
1118 : #endif /* HAVE_READLINK */
1119 : }
1120 0 : else if (S_ISDIR(statbuf.st_mode))
1121 : {
1122 0 : bool skip_this_dir = false;
1123 : ListCell *lc;
1124 :
1125 : /*
1126 : * Store a directory entry in the tar file so we can get the
1127 : * permissions right.
1128 : */
1129 0 : size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1130 : sizeonly);
1131 :
1132 : /*
1133 : * Call ourselves recursively for a directory, unless it happens
1134 : * to be a separate tablespace located within PGDATA.
1135 : */
1136 0 : foreach(lc, tablespaces)
1137 : {
1138 0 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1139 :
1140 : /*
1141 : * ti->rpath is the tablespace relative path within PGDATA, or
1142 : * NULL if the tablespace has been properly located somewhere
1143 : * else.
1144 : *
1145 : * Skip past the leading "./" in pathbuf when comparing.
1146 : */
1147 0 : if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1148 : {
1149 0 : skip_this_dir = true;
1150 0 : break;
1151 : }
1152 : }
1153 :
1154 : /*
1155 : * skip sending directories inside pg_tblspc, if not required.
1156 : */
1157 0 : if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1158 0 : skip_this_dir = true;
1159 :
1160 0 : if (!skip_this_dir)
1161 0 : size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks);
1162 : }
1163 0 : else if (S_ISREG(statbuf.st_mode))
1164 : {
1165 0 : bool sent = false;
1166 :
1167 0 : if (!sizeonly)
1168 0 : sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1169 : true);
1170 :
1171 0 : if (sent || sizeonly)
1172 : {
1173 : /* Add size, rounded up to 512byte block */
1174 0 : size += ((statbuf.st_size + 511) & ~511);
1175 0 : size += 512; /* Size of the header of the file */
1176 : }
1177 : }
1178 : else
1179 0 : ereport(WARNING,
1180 : (errmsg("skipping special file \"%s\"", pathbuf)));
1181 : }
1182 0 : FreeDir(dir);
1183 0 : return size;
1184 : }
1185 :
1186 : /*****
1187 : * Functions for handling tar file format
1188 : *
1189 : * Copied from pg_dump, but modified to work with libpq for sending
1190 : */
1191 :
1192 :
1193 : /*
1194 : * Given the member, write the TAR header & send the file.
1195 : *
1196 : * If 'missing_ok' is true, will not throw an error if the file is not found.
1197 : *
1198 : * Returns true if the file was successfully sent, false if 'missing_ok',
1199 : * and the file did not exist.
1200 : */
1201 : static bool
1202 0 : sendFile(char *readfilename, char *tarfilename, struct stat *statbuf,
1203 : bool missing_ok)
1204 : {
1205 : FILE *fp;
1206 : char buf[TAR_SEND_SIZE];
1207 : size_t cnt;
1208 0 : pgoff_t len = 0;
1209 : size_t pad;
1210 :
1211 0 : fp = AllocateFile(readfilename, "rb");
1212 0 : if (fp == NULL)
1213 : {
1214 0 : if (errno == ENOENT && missing_ok)
1215 0 : return false;
1216 0 : ereport(ERROR,
1217 : (errcode_for_file_access(),
1218 : errmsg("could not open file \"%s\": %m", readfilename)));
1219 : }
1220 :
1221 0 : _tarWriteHeader(tarfilename, NULL, statbuf, false);
1222 :
1223 0 : while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
1224 : {
1225 : /* Send the chunk as a CopyData message */
1226 0 : if (pq_putmessage('d', buf, cnt))
1227 0 : ereport(ERROR,
1228 : (errmsg("base backup could not send data, aborting backup")));
1229 :
1230 0 : len += cnt;
1231 0 : throttle(cnt);
1232 :
1233 0 : if (len >= statbuf->st_size)
1234 : {
1235 : /*
1236 : * Reached end of file. The file could be longer, if it was
1237 : * extended while we were sending it, but for a base backup we can
1238 : * ignore such extended data. It will be restored from WAL.
1239 : */
1240 0 : break;
1241 : }
1242 : }
1243 :
1244 : /* If the file was truncated while we were sending it, pad it with zeros */
1245 0 : if (len < statbuf->st_size)
1246 : {
1247 0 : MemSet(buf, 0, sizeof(buf));
1248 0 : while (len < statbuf->st_size)
1249 : {
1250 0 : cnt = Min(sizeof(buf), statbuf->st_size - len);
1251 0 : pq_putmessage('d', buf, cnt);
1252 0 : len += cnt;
1253 0 : throttle(cnt);
1254 : }
1255 : }
1256 :
1257 : /*
1258 : * Pad to 512 byte boundary, per tar format requirements. (This small
1259 : * piece of data is probably not worth throttling.)
1260 : */
1261 0 : pad = ((len + 511) & ~511) - len;
1262 0 : if (pad > 0)
1263 : {
1264 0 : MemSet(buf, 0, pad);
1265 0 : pq_putmessage('d', buf, pad);
1266 : }
1267 :
1268 0 : FreeFile(fp);
1269 :
1270 0 : return true;
1271 : }
1272 :
1273 :
1274 : static int64
1275 0 : _tarWriteHeader(const char *filename, const char *linktarget,
1276 : struct stat *statbuf, bool sizeonly)
1277 : {
1278 : char h[512];
1279 : enum tarError rc;
1280 :
1281 0 : if (!sizeonly)
1282 : {
1283 0 : rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1284 : statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1285 : statbuf->st_mtime);
1286 :
1287 0 : switch (rc)
1288 : {
1289 : case TAR_OK:
1290 0 : break;
1291 : case TAR_NAME_TOO_LONG:
1292 0 : ereport(ERROR,
1293 : (errmsg("file name too long for tar format: \"%s\"",
1294 : filename)));
1295 : break;
1296 : case TAR_SYMLINK_TOO_LONG:
1297 0 : ereport(ERROR,
1298 : (errmsg("symbolic link target too long for tar format: "
1299 : "file name \"%s\", target \"%s\"",
1300 : filename, linktarget)));
1301 : break;
1302 : default:
1303 0 : elog(ERROR, "unrecognized tar error: %d", rc);
1304 : }
1305 :
1306 0 : pq_putmessage('d', h, sizeof(h));
1307 : }
1308 :
1309 0 : return sizeof(h);
1310 : }
1311 :
1312 : /*
1313 : * Write tar header for a directory. If the entry in statbuf is a link then
1314 : * write it as a directory anyway.
1315 : */
1316 : static int64
1317 0 : _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
1318 : bool sizeonly)
1319 : {
1320 : /* If symlink, write it as a directory anyway */
1321 : #ifndef WIN32
1322 0 : if (S_ISLNK(statbuf->st_mode))
1323 : #else
1324 : if (pgwin32_is_junction(pathbuf))
1325 : #endif
1326 0 : statbuf->st_mode = S_IFDIR | S_IRWXU;
1327 :
1328 0 : return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1329 : }
1330 :
1331 : /*
1332 : * Increment the network transfer counter by the given number of bytes,
1333 : * and sleep if necessary to comply with the requested network transfer
1334 : * rate.
1335 : */
1336 : static void
1337 0 : throttle(size_t increment)
1338 : {
1339 : TimeOffset elapsed,
1340 : elapsed_min,
1341 : sleep;
1342 : int wait_result;
1343 :
1344 0 : if (throttling_counter < 0)
1345 0 : return;
1346 :
1347 0 : throttling_counter += increment;
1348 0 : if (throttling_counter < throttling_sample)
1349 0 : return;
1350 :
1351 : /* Time elapsed since the last measurement (and possible wake up). */
1352 0 : elapsed = GetCurrentTimestamp() - throttled_last;
1353 : /* How much should have elapsed at minimum? */
1354 0 : elapsed_min = elapsed_min_unit * (throttling_counter / throttling_sample);
1355 0 : sleep = elapsed_min - elapsed;
1356 : /* Only sleep if the transfer is faster than it should be. */
1357 0 : if (sleep > 0)
1358 : {
1359 0 : ResetLatch(MyLatch);
1360 :
1361 : /* We're eating a potentially set latch, so check for interrupts */
1362 0 : CHECK_FOR_INTERRUPTS();
1363 :
1364 : /*
1365 : * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1366 : * the maximum time to sleep. Thus the cast to long is safe.
1367 : */
1368 0 : wait_result = WaitLatch(MyLatch,
1369 : WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
1370 0 : (long) (sleep / 1000),
1371 : WAIT_EVENT_BASE_BACKUP_THROTTLE);
1372 :
1373 0 : if (wait_result & WL_LATCH_SET)
1374 0 : CHECK_FOR_INTERRUPTS();
1375 : }
1376 :
1377 : /*
1378 : * As we work with integers, only whole multiple of throttling_sample was
1379 : * processed. The rest will be done during the next call of this function.
1380 : */
1381 0 : throttling_counter %= throttling_sample;
1382 :
1383 : /*
1384 : * Time interval for the remaining amount and possible next increments
1385 : * starts now.
1386 : */
1387 0 : throttled_last = GetCurrentTimestamp();
1388 : }
|
