diff --git a/man/man3/pmiaddinstance.3 b/man/man3/pmiaddinstance.3 index 3c98312e99e..836af75e00d 100644 --- a/man/man3/pmiaddinstance.3 +++ b/man/man3/pmiaddinstance.3 @@ -45,6 +45,16 @@ pmiAddInstance($\fIindom\fP, $\fIinstance\fP, $\fIinst\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiAddInstance(\fIindom\fP, \fIinstance\fP, \fIinstid\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmiaddmetric.3 b/man/man3/pmiaddmetric.3 index 6d05e059879..11937790bfb 100644 --- a/man/man3/pmiaddmetric.3 +++ b/man/man3/pmiaddmetric.3 @@ -55,6 +55,23 @@ $\fIunits\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiAddMetric(\fIname\fP, +'in +\w'log.pmiAddMetric('u +\fIpmid\fP, +\fItype\fP, +\fIindom\fP, +\fIsem\fP, +\fIunits\fP) +.in +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmiend.3 b/man/man3/pmiend.3 index 50aa51cca72..68ff8c1dc8b 100644 --- a/man/man3/pmiend.3 +++ b/man/man3/pmiend.3 @@ -40,6 +40,16 @@ pmiEnd(); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiEnd() +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmigethandle.3 b/man/man3/pmigethandle.3 index 07f613d1215..fdde6cba0bd 100644 --- a/man/man3/pmigethandle.3 +++ b/man/man3/pmigethandle.3 @@ -40,6 +40,16 @@ $\fIhandle\fP = pmiGetHandle($\fIname\fP, $\fIinstance\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +\fIhandle\fP = log.pmiGetHandle(\fIname\fP, \fIinstance\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmiputlabel.3 b/man/man3/pmiputlabel.3 index 396586e692c..309106eb56b 100644 --- a/man/man3/pmiputlabel.3 +++ b/man/man3/pmiputlabel.3 @@ -54,6 +54,22 @@ $\fIvalue\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiPutLabel(\fItype\fP, +'in +\w'log.pmiPutLabel('u +\fIid\fP, +\fIinstance\fP, +\fIname\fP, +\fIvalue\fP) +.in +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmiputmark.3 b/man/man3/pmiputmark.3 index 7f82d17470f..c915675c8dc 100644 --- a/man/man3/pmiputmark.3 +++ b/man/man3/pmiputmark.3 @@ -40,6 +40,16 @@ pmiPutMark(); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiPutMark() +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmiputresult.3 b/man/man3/pmiputresult.3 index f6de5458b1e..992ba8e731d 100644 --- a/man/man3/pmiputresult.3 +++ b/man/man3/pmiputresult.3 @@ -16,7 +16,8 @@ .\" .TH PMIPUTRESULT 3 "" "Performance Co-Pilot" .SH NAME -\f3pmiPutResult\f1 \- add a data record to a LOGIMPORT archive +\f3pmiPutResult\f1, +\f3pmiPutHighResResult\f1 \- add a data record to a LOGIMPORT archive .SH "C SYNOPSIS" .ft 3 .ad l @@ -25,52 +26,76 @@ .br #include .sp -int pmiPutResult(const pmResult *\fIresult\fP); +int pmiPutResult(const pmResult_v2 *\fIresult\fP); +.br +int pmiPutHighResResult(const pmResult *\fIresult\fP); .sp cc ... \-lpcp_import \-lpcp .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.put_result(\fIresult\fP) +.br +log.put_highres_result(\fIresult\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), -.B pmiPutResult -provides an interface for developers familiar with the internal +these routines provide an interface for developers familiar with the internal PCP data structures to create output archives directly. .PP -By building the +By building a .B pmResult -data structure directly, then calling -.B pmiPutResult -the developer avoids calls to +data structure directly and calling one of these routines, the developer +avoids the separate .BR pmiPutValue (3) and/or .BR pmiPutValueHandle (3) -followed by a call to -.BR pmiWrite (3) +calls followed by +.BR pmiHighResWrite (3) for each record written to the archive. .PP -Any metrics and instances appearing in the +.B pmiPutHighResResult +is the preferred interface. +It accepts a +.B pmResult +with a +.B struct timespec +timestamp, giving nanosecond resolution and Y2038-safe 64-bit seconds. +.PP +.B pmiPutResult +accepts the older +.B pmResult_v2 +type with a +.B struct timeval +timestamp (microsecond resolution). +.PP +Any metrics and instances appearing in .I result must have been defined by prior calls to .BR pmiAddMetric (3) and .BR pmiAddInstance (3). -.PP -.B pmiPutResult -will arrange for any new metadata (metrics and/or instance domain changes) -covered by +Both routines will arrange for any new metadata (metrics and/or instance +domain changes) covered by .I result -to be also written to the PCP archive. +to be written to the PCP archive. .PP Because of the complexity of the .B pmResult -data structure, this routine is not available in the Perl +data structure, neither routine is available in the Perl interface to the LOGIMPORT services. .SH DIAGNOSTICS -.B pmiPutResult -returns zero on success else a negative value that can be turned into an -error message by calling +Both routines return zero on success else a negative value that can be +turned into an error message by calling .BR pmiErrStr (3). .SH SEE ALSO .BR LOGIMPORT (3), @@ -82,6 +107,7 @@ error message by calling .BR pmiPutText (3), .BR pmiPutLabel (3), .BR pmiPutValueHandle (3), -.BR pmiSetTimezone (3) +.BR pmiSetTimezone (3), +.BR pmiHighResWrite (3) and .BR pmiWrite (3). diff --git a/man/man3/pmiputtext.3 b/man/man3/pmiputtext.3 index 55d3db397be..c65339ed1db 100644 --- a/man/man3/pmiputtext.3 +++ b/man/man3/pmiputtext.3 @@ -45,6 +45,16 @@ pmiPutText($\fItype\fP, $\fIclass\fP, $\fIid\fP, $\fIcontent\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiPutText(\fItype\fP, \fIcls\fP, \fIid\fP, \fIcontent\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmiputvalue.3 b/man/man3/pmiputvalue.3 index f16128fdc1a..8c8a596a5a3 100644 --- a/man/man3/pmiputvalue.3 +++ b/man/man3/pmiputvalue.3 @@ -45,6 +45,16 @@ pmiPutValue($\fIname\fP, $\fIinstance\fP, $\fIvalue\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiPutValue(\fIname\fP, \fIinstance\fP, \fIvalue\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmiputvaluehandle.3 b/man/man3/pmiputvaluehandle.3 index 815f7ed0afe..db2c108df7b 100644 --- a/man/man3/pmiputvaluehandle.3 +++ b/man/man3/pmiputvaluehandle.3 @@ -1,7 +1,7 @@ '\"macro stdmacro .\" .\" Copyright (c) 2010 Ken McDonell. All Rights Reserved. -.\" Copyright (c) 2018 Red Hat. +.\" Copyright (c) 2018,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -16,6 +16,7 @@ .\" .TH PMIPUTVALUEHANDLE 3 "" "Performance Co-Pilot" .SH NAME +\f3pmiPutAtomValueHandle\f1, \f3pmiPutValueHandle\f1 \- add a value for a metric-instance pair via a handle .SH "C SYNOPSIS" .ft 3 @@ -25,6 +26,8 @@ .br #include .sp +int pmiPutAtomValueHandle(int \fIhandle\fP, pmAtomValue *\fIatom\fP); +.br int pmiPutValueHandle(int \fIhandle\fP, const char *\fIvalue\fP); .sp cc ... \-lpcp_import \-lpcp @@ -41,17 +44,37 @@ pmiPutValueHandle($\fIhandle\fP, $\fIvalue\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiPutValueHandle(\fIhandle\fP, \fIvalue\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), +.B pmiPutAtomValueHandle +and .B pmiPutValueHandle -adds a single value to the current output record for a given +add a single value to the current output record for a given metric and instance, using the .I handle defined by an earlier call to .BR pmiGetHandle (3). .PP -The +With +.B pmiPutAtomValueHandle +the +.I atom +passes the binary value directly using a union of types. +.PP +In the case of +.B pmiPutValueHandle +the .I value should be in a format consistent with the metric's type as defined in the call to @@ -60,6 +83,7 @@ defined in the call to No data will be written until .BR pmiWrite (3) is called, so multiple calls to +.BR pmiPutAtomValueHandle , .B pmiPutValueHandle or .BR pmiPutValue (3) @@ -67,8 +91,10 @@ are typically used to accumulate data values for several metric-instance pairs before calling .BR pmiWrite (3). .SH DIAGNOSTICS +.B pmiPutAtomValueHandle +and .B pmiPutValueHandle -returns zero on success else a negative value that can be turned into an +return zero on success else a negative value that can be turned into an error message by calling .BR pmiErrStr (3). .SH SEE ALSO diff --git a/man/man3/pmisethostname.3 b/man/man3/pmisethostname.3 index a87e98604fb..d161d8c7b22 100644 --- a/man/man3/pmisethostname.3 +++ b/man/man3/pmisethostname.3 @@ -40,6 +40,16 @@ pmiSetHostname($\fIvalue\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiSetHostname(\fIvalue\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmisettimezone.3 b/man/man3/pmisettimezone.3 index 8bf9166e121..88c937d2183 100644 --- a/man/man3/pmisettimezone.3 +++ b/man/man3/pmisettimezone.3 @@ -40,6 +40,16 @@ pmiSetTimezone($\fIvalue\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiSetTimezone(\fIvalue\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmisetversion.3 b/man/man3/pmisetversion.3 index bd2313096d7..7e51655a746 100644 --- a/man/man3/pmisetversion.3 +++ b/man/man3/pmisetversion.3 @@ -16,7 +16,7 @@ .TH PMISETVERSION 3 "" "Performance Co-Pilot" .SH NAME \f3pmiSetVersion\f1 \- set the archive version for a LOGIMPORT archive -.SH C SYNOPSIS +.SH "C SYNOPSIS" .ft 3 .ad l .hy 0 @@ -30,7 +30,7 @@ cc ... \-lpcp_import \-lpcp .hy .ad .ft 1 -.SH PERL SYNOPSIS +.SH "Perl SYNOPSIS" .ft 3 .ad l .hy 0 @@ -40,13 +40,13 @@ pmiSetVersion($\fIvalue\fP); .hy .ad .ft 1 -.SH PYTHON SYNOPSIS +.SH "Python SYNOPSIS" .ft 3 .ad l .hy 0 from pcp import pmi .sp -log = pmi.pmiLogImport(\fIpath\fP).pmiSetVersion(\fIvalue\fP) +log.pmiSetVersion(\fIvalue\fP) .hy .ad .ft 1 diff --git a/man/man3/pmisetvolumesize.3 b/man/man3/pmisetvolumesize.3 new file mode 100644 index 00000000000..471af7c67c2 --- /dev/null +++ b/man/man3/pmisetvolumesize.3 @@ -0,0 +1,124 @@ +'\"macro stdmacro +.\" +.\" Copyright (c) 2026 Red Hat. +.\" +.\" This program is free software; you can redistribute it and/or modify it +.\" under the terms of the GNU General Public License as published by the +.\" Free Software Foundation; either version 2 of the License, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +.\" or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +.\" for more details. +.\" +.\" +.TH PMISETVOLUMESIZE 3 "" "Performance Co-Pilot" +.SH NAME +\f3pmiSetVolumeSize\f1 \- configure automatic data volume rotation for a LOGIMPORT archive +.SH "C SYNOPSIS" +.ft 3 +.ad l +.hy 0 +#include +.br +#include +.sp +int pmiSetVolumeSize(size_t \fImax_bytes\fP, void (*\fIon_rotate\fP)(const char *)); +.sp +cc ... \-lpcp_import \-lpcp +.hy +.ad +.ft 1 +.SH DESCRIPTION +As part of the Performance Co-Pilot Log Import API (see +.BR LOGIMPORT (3)), +.B pmiSetVolumeSize +configures automatic data volume rotation for the archive associated +with the current context. +.PP +After each successful call to +.BR pmiHighResWrite (3) +or +.BR pmiWrite (3), +if the current data volume file size meets or exceeds +.IR max_bytes , +the library closes the current data volume, opens the next numbered +volume, and writes a new volume label. +Volume files are named +.IB archive .0 , +.IB archive .1 , +and so on, where +.I archive +is the base path given to +.BR pmiStart (3). +The +.I .meta +and +.I .index +files are shared across all volumes and remain open throughout. +.PP +If +.I on_rotate +is not NULL, it is called immediately after the old volume is closed +and before the function returns. +The single argument is the full path of the just-closed volume file +(e.g.\& +.IR /var/log/sa/pcp16/pcp16.0 ). +The caller may use this callback to compress the completed volume or +perform any other post-rotation work. +The callback is invoked synchronously; long-running operations should +be deferred to a child process. +.PP +Passing +.I max_bytes +as zero disables volume rotation (the default). +.PP +.B pmiSetVolumeSize +interacts with the existing +.B PCP_LOGIMPORT_MAXLOGSZ +environment variable: both mechanisms trigger +.BR newvolume () +independently, so if both are active the smaller threshold governs. +.SH EXAMPLE +The following fragment opens a PCP archive, configures 100\ MB +volume rotation, and compresses each completed volume: +.PP +.nf +.ft 3 +static void +on_vol(const char *path) +{ + char cmd[MAXPATHLEN + 32]; + pmsprintf(cmd, sizeof(cmd), "xz %s &", path); + system(cmd); +} + +pmiStart("/var/log/sa/pcp16/pcp16", 0); +pmiSetVolumeSize(100 * 1024 * 1024, on_vol); +.ft 1 +.fi +.SH DIAGNOSTICS +.B pmiSetVolumeSize +returns zero on success. +If there is no current context, +.B PM_ERR_NOCONTEXT +is returned. +If +.I max_bytes +is greater than zero but does not exceed the on-disk label size for the +configured archive version (124 bytes for v2, 800 bytes for v3), +.B PM_ERR_CONV +is returned. +A threshold at or below the label size would cause the new volume's label +to immediately exceed the threshold on every write, triggering a +rotation cascade. +.SH SEE ALSO +.BR LOGIMPORT (3), +.BR pmiEnd (3), +.BR pmiErrStr (3), +.BR pmiHighResWrite (3), +.BR pmiStart (3), +.BR pmiWrite (3) +and +.BR pmlogcheck (1). diff --git a/man/man3/pmistart.3 b/man/man3/pmistart.3 index 99f73f75c1a..79bb69e3525 100644 --- a/man/man3/pmistart.3 +++ b/man/man3/pmistart.3 @@ -25,7 +25,7 @@ .br #include .sp -int pmiStart(const char *\fIarchive\fP, int \fIinherit\fP); +int pmiStart(const char *\fIarchive\fP, int \fIflags\fP); .sp cc ... \-lpcp_import \-lpcp .hy @@ -37,7 +37,17 @@ cc ... \-lpcp_import \-lpcp .hy 0 use PCP::LogImport; .sp -pmiStart($\fIarchive\fP, $\fIinherit\fP); +pmiStart($\fIarchive\fP, $\fIflags\fP); +.hy +.ad +.ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log = pmi.pmiLogImport(\fIarchive\fP, \fIflags\fP) .hy .ad .ft 1 @@ -88,20 +98,78 @@ by calls to or .BR pmiPutValueHandle (3). .PP -If -.I inherit -is true, then the new context will inherit any and all -metadata (metrics, instance domains, instances and handles) from the current -context, otherwise the new context is created with no metadata. +The +.I flags +argument is a bitwise-OR of zero or more of the following constants: +.TP +.B PMI_INHERIT +The new context will inherit any and all metadata (metrics, instance +domains, instances and handles) from the current context. The basename for the output PCP archive, the source hostname, the source timezone and any data values from the current context are .B not inherited. If this is the first call to .B pmiStart -the metadata will be empty -independent of the value of -.IR inherit . +the metadata will be empty independent of whether +.B PMI_INHERIT +is set. +.TP +.B PMI_APPEND +Open an existing PCP archive for appending rather than creating a +new one. +The archive label (hostname, timezone, version, start timestamp) is +read from the existing +.IR archive .meta +file and used to initialise the new context. +The temporal index is read to seed the last-written timestamp; all +subsequent +.BR pmiWrite (3) +calls must use timestamps strictly later than this value. +The +.IR archive .meta +and +.IR archive .index +files are opened in read/write mode and positioned at end-of-file; +the highest-numbered data volume is opened in append mode. +Metrics and instance domains must still be registered via +.BR pmiAddMetric (3) +and +.BR pmiAddInstance (3) +before calling +.BR pmiPutValue (3) +\(em if the metric descriptor is already present in the archive the +duplicate entry in +.I archive .meta +is harmless and will be ignored by readers. +If the archive files do not yet exist, +.B PMI_APPEND +behaves identically to creating a new archive (no flags). +.PP +.B PMI_APPEND +and +.B PMI_INHERIT +may be used together. +The inherited metric and instance-domain definitions are carried into +the new context, and when the archive is opened on the first +.BR pmiWrite (3) +call, each inherited descriptor is checked against what is already +on disk: compatible descriptors are silently skipped (no duplicate +written); incompatible ones are rejected with an appropriate error +(e.g. +.BR PM_ERR_LOGCHANGETYPE ). +When the previous context was writing to the same archive, every +inherited descriptor already exists on disk so the combination is +effectively a no-op beyond convenience. +.PP +If +.I flags +is zero, or neither +.B PMI_INHERIT +nor +.B PMI_APPEND +is set, the new context is created with no metadata and a new archive +will be created from scratch. .PP Since no physical files for the output PCP archive will be created until the first call to @@ -125,7 +193,10 @@ call or a call to or a call to .BR pmiEnd (3). .SH DIAGNOSTICS -It is an error if the physical files +When creating a new archive (neither +.B PMI_APPEND +nor any other flag that implies file creation), it is an error if the +physical files \fIarchive\fR.\fB0\fR and/or \fIarchive\fR.\fBindex\fR and/or \fIarchive\fR.\fBmeta\fR already exist, but this is not discovered @@ -136,6 +207,14 @@ or so .B pmiStart always returns a positive context identifier. +.PP +When +.B PMI_APPEND +is set and the archive files do not exist, +.B pmiStart +silently falls back to creating a new archive, making it safe to use +.B PMI_APPEND +unconditionally on the first invocation of a data collector. .SH SEE ALSO .BR LOGIMPORT (3), .BR PMAPI (3), diff --git a/man/man3/pmiunits.3 b/man/man3/pmiunits.3 index 0e1ea38b084..ee5bb768ae7 100644 --- a/man/man3/pmiunits.3 +++ b/man/man3/pmiunits.3 @@ -88,6 +88,23 @@ $\fIextraScale\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +\fIpmid\fP = pmi.pmiLogImport.pmiID(\fIdomain\fP, \fIcluster\fP, \fIitem\fP) +.br +\fIindom\fP = pmi.pmiLogImport.pmiInDom(\fIdomain\fP, \fIserial\fP) +.br +\fIunits\fP = pmi.pmiLogImport.pmiUnits(\fIdim_space\fP, \fIdim_time\fP, \fIdim_count\fP, +'in +\w'pmi.pmiLogImport.pmiUnits('u +\fIscale_space\fP, \fIscale_time\fP, \fIscale_count\fP) +.in +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), diff --git a/man/man3/pmiwrite.3 b/man/man3/pmiwrite.3 index ef2a4c03455..ee5b9193344 100644 --- a/man/man3/pmiwrite.3 +++ b/man/man3/pmiwrite.3 @@ -1,7 +1,7 @@ '\"macro stdmacro .\" .\" Copyright (c) 2010 Ken McDonell. All Rights Reserved. -.\" Copyright (c) 2018 Red Hat. +.\" Copyright (c) 2018,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -16,7 +16,9 @@ .\" .TH PMIWRITE 3 "" "Performance Co-Pilot" .SH NAME -\f3pmiWrite\f1 \- flush data to a LOGIMPORT archive +\f3pmiWrite\f1, +\f3pmiWrite2\f1, +\f3pmiHighResWrite\f1 \- flush data to a LOGIMPORT archive .SH "C SYNOPSIS" .ft 3 .ad l @@ -26,6 +28,10 @@ #include .sp int pmiWrite(int \fIsec\fP, int \fIusec\fP); +.br +int pmiWrite2(int64_t \fIsec\fP, int \fIusec\fP); +.br +int pmiHighResWrite(int64_t \fIsec\fP, int \fInsec\fP); .sp cc ... \-lpcp_import \-lpcp .hy @@ -41,11 +47,22 @@ pmiWrite($\fIsec\fP, $\fIusec\fP); .hy .ad .ft 1 +.SH "Python SYNOPSIS" +.ft 3 +.ad l +.hy 0 +from pcp import pmi +.sp +log.pmiWrite(\fIsec\fP, \fIusec\fP) +.br +log.pmiHighResWrite(\fIsec\fP, \fInsec\fP) +.hy +.ad +.ft 1 .SH DESCRIPTION As part of the Performance Co-Pilot Log Import API (see .BR LOGIMPORT (3)), -.B pmiWrite -forces accumulated data values out to the PCP archive. +these routines flush accumulated data values out to the PCP archive. .PP The data values and associated metadata have previously been built up using calls to @@ -55,17 +72,46 @@ built up using calls to and .BR pmiPutValueHandle (3). .PP -The current set of data values and any new metadata is written to the archive with -a timestamp of +The current set of data values and any new metadata is written to the archive +with the specified timestamp in the source timezone of the archive +(see +.BR pmiSetTimezone (3)). +.PP +.B pmiHighResWrite +is the preferred interface. +It accepts a nanosecond-resolution timestamp: seconds since the Unix epoch +as a 64-bit signed integer +.RI ( sec ) +and nanoseconds within that second +.RI ( nsec ). +Using a 64-bit .I sec -and -.I usec -in the source timezone of the archive, see -.BR pmiSetTimezone (3). -.SH DIAGNOSTICS +argument makes +.B pmiHighResWrite +safe beyond the year 2038, unlike +.BR pmiWrite . +.PP +.B pmiWrite2 +is equivalent to +.B pmiHighResWrite +except that the sub-second component is specified in microseconds +rather than nanoseconds. +.PP .B pmiWrite -returns zero on success else a negative value that can be turned into an -error message by calling +is deprecated. +Its 32-bit +.I sec +argument overflows in January 2038 and callers should migrate to +.B pmiHighResWrite +(or +.BR pmiWrite2 ). +Passing a negative value for +.I sec +to any of these routines causes the current wall-clock time to be +used as the timestamp. +.SH DIAGNOSTICS +All three routines return zero on success, else a negative value that +can be turned into an error message by calling .BR pmiErrStr (3). .SH SEE ALSO .BR LOGIMPORT (3), diff --git a/qa/1418.out b/qa/1418.out index 891d27d249f..a1cda24719e 100644 --- a/qa/1418.out +++ b/qa/1418.out @@ -40,9 +40,11 @@ pmiPutValue: OK pmiPutValue: OK pmiPutValue: OK pmiGetHandle: OK ->1 +pmiGetHandle: OK ->2 pmiGetHandle: Error: Unknown metric name pmiPutValueHandle: Error: Value already assigned for this metric-instance pmiPutValueHandle: Error: Illegal handle +pmiPutAtomValueHandle: Error: Illegal handle pmiPutText: OK pmiPutText: OK pmiPutText: OK @@ -124,6 +126,7 @@ pmiDump: context 0 of 1 archive: myarchive instance[1] eek (2) instance[2] blah (3) handle[0] metric=my.metric.foo (245.0.1) instance=-1 + handle[1] metric=my.metric.long (245.0.3) instance=-1 text[0] pmid=245.0.1 [One line text for my.metric.foo] text[1] pmid=245.0.1 Full help text for my.metric.foo @@ -170,13 +173,17 @@ pmiWrite: OK pmiPutValue: OK pmiWrite: OK pmiWrite: Error: No data to output +pmiPutAtomValueHandle: OK +pmiPutAtomValueHandle: OK +pmiPutAtomValueHandle: Error: Value already assigned for this metric-instance +pmiWrite: OK pmiPutMark: OK pmiEnd: OK pmiStart: OK ->2 pmiAddInstance: Error: Internal instance identifer already defined -pmiGetHandle: OK ->2 +pmiGetHandle: OK ->3 pmiPutValueHandle: OK -pmiPutValueHandle: Error: Illegal handle +pmiPutValueHandle: Error: Value already assigned for this metric-instance pmiDump: context 1 of 2 archive: myotherarchive state: 1 (start) hostname: timezone: metric[0] name=my.metric.foo pmid=245.0.1 @@ -205,7 +212,8 @@ pmiDump: context 1 of 2 archive: myotherarchive instance[1] eek (2) instance[2] blah (3) handle[0] metric=my.metric.foo (245.0.1) instance=-1 - handle[1] metric=my.metric.bar (245.0.2) instance=2 + handle[1] metric=my.metric.long (245.0.3) instance=-1 + handle[2] metric=my.metric.bar (245.0.2) instance=2 text[0] pmid=245.0.1 [One line text for my.metric.foo] text[1] pmid=245.0.1 Full help text for my.metric.foo @@ -290,7 +298,7 @@ TIMESTAMP 3 instances Temporal Index Log Vol end(meta) end(log) TIMESTAMP 0 808 808 -TIMESTAMP 0 2385 1256 +TIMESTAMP 0 2385 1332 [260 bytes] TIMESTAMP 7 metrics @@ -313,6 +321,11 @@ TIMESTAMP 2 metrics TIMESTAMP 1 metric 245.0.5 (my.metric.string): value "a third string value" +[76 bytes] +TIMESTAMP 2 metrics + 245.0.1 (my.metric.foo): value 55555 + 245.0.3 (my.metric.long): value 1234567890123 + [24 bytes] TIMESTAMP diff --git a/qa/1693 b/qa/1693 new file mode 100755 index 00000000000..9f79aff8134 --- /dev/null +++ b/qa/1693 @@ -0,0 +1,56 @@ +#!/bin/sh +# PCP QA Test No. 1693 +# Exercise PMI_APPEND flag to pmiStart() - create an archive then +# append additional records in a separate context. +# +# Copyright (c) 2026 Red Hat. All Rights Reserved. +# + +seq=`basename $0` +echo "QA output created by $seq" + +. ./common.product +. ./common.filter +. ./common.check + +[ -f ${PCP_LIB_DIR}/libpcp_import.${DSO_SUFFIX} ] || \ + _notrun "No support for libpcp_import" + +status=0 # success is the default! +trap "cd $here; rm -rf $tmp; exit \$status" 0 1 2 3 15 + +mkdir $tmp +cd $tmp + +# Handles both the pmi* stderr output and pmdumplog output. +_filter() +{ + sed -e '/pmResult/s/ .* numpmid/ ... numpmid/' \ + | _filter_pmdumplog +} + +# real QA test starts here +$here/src/check_import_append 2>&1 | _filter + +echo +echo "=== archive integrity ===" +pmlogcheck appendtest + +echo +echo "=== all four records present ===" +pmdumplog -t appendtest 2>&1 | _filter + +echo +echo "=== fallback archive created ===" +pmlogcheck appendnew + +echo +echo "=== Phase 4: pmiSetVolumeSize ===" +$here/src/check_volsize 2>&1 + +echo +echo "=== multi-volume archive integrity ===" +pmlogcheck volsizetest + +# success, all done +exit diff --git a/qa/1693.out b/qa/1693.out new file mode 100644 index 00000000000..804541c1054 --- /dev/null +++ b/qa/1693.out @@ -0,0 +1,64 @@ +QA output created by 1693 +=== Phase 1: create === +pmiStart: OK ->1 +pmiSetHostname: OK +pmiSetTimezone: OK +pmiAddMetric counter: OK +pmiAddMetric instant: OK +pmiPutValue counter (1): OK +pmiPutValue instant (1): OK +pmiWrite (1): OK +pmiPutValue counter (2): OK +pmiPutValue instant (2): OK +pmiWrite (2): OK +pmiEnd: OK + +=== Phase 2: append === +pmiStart (PMI_APPEND): OK ->2 +pmiAddMetric counter (append): OK +pmiAddMetric instant (append): OK +pmiPutValue counter (3): OK +pmiPutValue instant (3): OK +pmiWrite (3): OK +pmiPutValue counter (4): OK +pmiPutValue instant (4): OK +pmiWrite (4): OK +pmiEnd (append): OK + +=== Phase 3: append-creates-new === +pmiStart (PMI_APPEND, new archive): OK ->3 +pmiAddMetric counter (new): OK +pmiPutValue counter (new): OK +pmiWrite (new): OK +pmiEnd (new): OK + +=== archive integrity === + +=== all four records present === + +Temporal Index + Log Vol end(meta) end(log) +TIMESTAMP 0 808 808 +TIMESTAMP 0 922 960 +TIMESTAMP 0 922 1112 + +=== fallback archive created === + +=== Phase 4: pmiSetVolumeSize === +pmiStart: OK ->1 +pmiSetHostname: OK +pmiSetTimezone: OK +pmiAddMetric: OK +pmiGetHandle: OK ->1 +pmiSetVolumeSize: OK +pmiHighResWrite loop: OK +pmiEnd: OK +volume rotation: OK +volsizetest.0: present +volsizetest.1: present +volume size bound: OK +callback path: OK +label size guard: OK +pmiSetVolumeSize(0) disable: OK + +=== multi-volume archive integrity === diff --git a/qa/369.out b/qa/369.out index a3e20de9a92..e36b7aac9bf 100644 --- a/qa/369.out +++ b/qa/369.out @@ -40,9 +40,11 @@ pmiPutValue: OK pmiPutValue: OK pmiPutValue: OK pmiGetHandle: OK ->1 +pmiGetHandle: OK ->2 pmiGetHandle: Error: Unknown metric name pmiPutValueHandle: Error: Value already assigned for this metric-instance pmiPutValueHandle: Error: Illegal handle +pmiPutAtomValueHandle: Error: Illegal handle pmiPutText: OK pmiPutText: OK pmiPutText: OK @@ -124,6 +126,7 @@ pmiDump: context 0 of 1 archive: myarchive instance[1] eek (2) instance[2] blah (3) handle[0] metric=my.metric.foo (245.0.1) instance=-1 + handle[1] metric=my.metric.long (245.0.3) instance=-1 text[0] pmid=245.0.1 [One line text for my.metric.foo] text[1] pmid=245.0.1 Full help text for my.metric.foo @@ -170,13 +173,17 @@ pmiWrite: OK pmiPutValue: OK pmiWrite: OK pmiWrite: Error: No data to output +pmiPutAtomValueHandle: OK +pmiPutAtomValueHandle: OK +pmiPutAtomValueHandle: Error: Value already assigned for this metric-instance +pmiWrite: OK pmiPutMark: OK pmiEnd: OK pmiStart: OK ->2 pmiAddInstance: Error: Internal instance identifer already defined -pmiGetHandle: OK ->2 +pmiGetHandle: OK ->3 pmiPutValueHandle: OK -pmiPutValueHandle: Error: Illegal handle +pmiPutValueHandle: Error: Value already assigned for this metric-instance pmiDump: context 1 of 2 archive: myotherarchive state: 1 (start) hostname: timezone: metric[0] name=my.metric.foo pmid=245.0.1 @@ -205,7 +212,8 @@ pmiDump: context 1 of 2 archive: myotherarchive instance[1] eek (2) instance[2] blah (3) handle[0] metric=my.metric.foo (245.0.1) instance=-1 - handle[1] metric=my.metric.bar (245.0.2) instance=2 + handle[1] metric=my.metric.long (245.0.3) instance=-1 + handle[2] metric=my.metric.bar (245.0.2) instance=2 text[0] pmid=245.0.1 [One line text for my.metric.foo] text[1] pmid=245.0.1 Full help text for my.metric.foo @@ -290,7 +298,7 @@ TIMESTAMP 3 instances Temporal Index Log Vol end(meta) end(log) TIMESTAMP 0 132 132 -TIMESTAMP 0 1677 564 +TIMESTAMP 0 1677 636 [256 bytes] TIMESTAMP 7 metrics @@ -313,6 +321,11 @@ TIMESTAMP 2 metrics TIMESTAMP 1 metric 245.0.5 (my.metric.string): value "a third string value" +[72 bytes] +TIMESTAMP 2 metrics + 245.0.1 (my.metric.foo): value 55555 + 245.0.3 (my.metric.long): value 1234567890123 + [20 bytes] TIMESTAMP diff --git a/qa/708.out b/qa/708.out index e13993394dd..800a0c54242 100644 --- a/qa/708.out +++ b/qa/708.out @@ -5,6 +5,7 @@ pmiSetTimezone: UTC pmid 251.1.1 units pmiAddMetric: qa.one +pmiPutValue: qa.one (epoch) pmiPutValue: qa.one pmid 251.1.2 indom 251.3 @@ -26,7 +27,7 @@ Note: timezone set to local timezone of host "fu.bar.com" from archive qa.one Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0x........ Semantics: discrete Units: none - value 42 + value 1 qa.two Data Type: float InDom: 251.3 0x........ diff --git a/qa/group b/qa/group index d2b4d9a8896..234f20438eb 100644 --- a/qa/group +++ b/qa/group @@ -2238,7 +2238,7 @@ suse 1690 pmseries local 1691 pmseries local 1692 pmda.pmcd local -1693:reserved libpcp_import python +1693 libpcp_import local 1694 pidstat local python pcp pmlogextract 1695 pmproxy valgrind local 1696 pmproxy valgrind local diff --git a/qa/src/.gitignore b/qa/src/.gitignore index 38ec075fac5..4382406cc82 100644 --- a/qa/src/.gitignore +++ b/qa/src/.gitignore @@ -30,10 +30,12 @@ chain check_attribute check_fault_injection check_import +check_import_append check_import_name check_import.pl check_pmiend_fdleak check_pmi_errconv +check_volsize checkstructs chkacc1 chkacc2 diff --git a/qa/src/GNUlocaldefs b/qa/src/GNUlocaldefs index c6dd709cf40..5bf6b624ed4 100644 --- a/qa/src/GNUlocaldefs +++ b/qa/src/GNUlocaldefs @@ -1,5 +1,5 @@ # -# Copyright (c) 2012-2021 Red Hat. +# Copyright (c) 2012-2026 Red Hat. # Copyright (c) 2009 Aconex. All Rights Reserved. # Copyright (c) 1997-2002 Silicon Graphics, Inc. All Rights Reserved. # @@ -36,7 +36,8 @@ CFILES = disk_test.c exercise.c context_test.c chkoptfetch.c \ mmv2_genstats.c mmv2_instances.c mmv2_nostats.c mmv2_simple.c \ mmv3_simple.c mmv3_labels.c mmv3_bad_labels.c mmv3_nostats.c mmv3_genstats.c \ record.c record-setarg.c clientid.c grind_ctx.c \ - pmdacache.c check_import.c unpack.c hrunpack.c aggrstore.c atomstr.c \ + check_import_append.c check_import_name.c check_import.c check_volsize.c \ + pmdacache.c unpack.c hrunpack.c aggrstore.c atomstr.c \ semstr.c grind_conv.c getconfig.c err.c torture_logmeta.c keycache.c \ keycache2.c pmdaqueue.c drain-server.c template.c anon-sa.c \ username.c rtimetest.c getcontexthost.c badpmda.c chklogputresult.c \ @@ -56,7 +57,13 @@ CFILES = disk_test.c exercise.c context_test.c chkoptfetch.c \ throttle.c throttle_timeout.c y2038.c bigpmcdpmids.c pdu-gadget.c \ strnfoo.c mmv_ondisk.c newcontext.c api_abi.c interp_bug3.c \ pmsetmode.c scanindex.c localtime.c httpcache.c unregister.c \ - genpmns.c undelta.c series_time_parse_test.c derived_control.c + genpmns.c undelta.c series_time_parse_test.c derived_control.c \ + multithread0.c multithread1.c multithread2.c multithread3.c \ + multithread4.c multithread5.c multithread6.c multithread7.c \ + multithread8.c multithread9.c multithread10.c multithread11.c \ + multithread12.c multithread13.c multithread14.c \ + exerlock.c hashwalk.c parsehostattrs.c parsehostspec.c getoptions.c + ifeq ($(shell test -f ../localconfig && echo 1), 1) include ../localconfig @@ -68,53 +75,7 @@ $(error Cannot make qa/localconfig) endif endif -ifeq ($(shell test $(PCP_VER) -ge 3600 && echo 1), 1) -CFILES += multithread0.c multithread1.c multithread2.c multithread3.c \ - multithread4.c multithread5.c multithread6.c multithread7.c \ - multithread8.c multithread9.c multithread10.c multithread11.c \ - multithread12.c multithread13.c multithread14.c \ - exerlock.c -else -MYFILES += multithread0.c multithread1.c multithread2.c multithread3.c \ - multithread4.c multithread5.c multithread6.c multithread7.c \ - multithread8.c multithread9.c multithread10.c multithread11.c \ - multithread12.c multithread13.c multithread14.c \ - exerlock.c -LDIRT += multithread0 multithread1 multithread2 multithread3 \ - multithread4 multithread5 multithread6 multithread7 \ - multithread8 multithread9 multithread10 multithread11 \ - multithread12 multithread13 multithread14 \ - exerlock -endif - -ifeq ($(shell test $(PCP_VER) -ge 3700 && echo 1), 1) -CFILES += hashwalk.c -else -MYFILES += hashwalk.c -LDIRT += hashwalk -endif - -ifeq ($(shell test $(PCP_VER) -ge 3800 && echo 1), 1) -CFILES += parsehostattrs.c parsehostspec.c check_import_name.c -else -MYFILES += parsehostattrs.c parsehostspec.c check_import_name.c -LDIRT += parsehostattrs parsehostspec check_import_name -endif - -ifeq ($(shell test $(PCP_VER) -ge 3802 && echo 1), 1) -POSIXFILES += chkacc4.c -else -MYFILES += chkacc4.c -LDIRT += chkacc4 -endif - -ifeq ($(shell test $(PCP_VER) -ge 3901 && echo 1), 1) -CFILES += getoptions.c XTRATARGETS += getoptions_v2 -else -MYFILES += getoptions.c -LDIRT += getoptions getoptions_v2 -endif ifeq ($(shell test -f /usr/include/pcp/fault.h && echo 1), 1) # only make these ones if the fault injection version of libpcp @@ -142,7 +103,7 @@ endif # POSIXFILES = \ ipc.c proc_test.c context_fd_leak.c arch_maxfd.c torture_trace.c \ - 779246.c killparent.c fetchloop.c chain.c spawn.c + 779246.c killparent.c fetchloop.c chain.c spawn.c chkacc4.c TRACEFILES = \ obs.c tstate.c tabort.c @@ -1434,22 +1395,30 @@ sortinst: sortinst.c check_import: check_import.c rm -f $@ - $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_import + $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_archive -lpcp_import $(LINKER_MAKERULE) check_import_name: check_import_name.c rm -f $@ - $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_import + $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_archive -lpcp_import $(LINKER_MAKERULE) +check_import_append: check_import_append.c + rm -f $@ + $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_archive -lpcp_import + +check_volsize: check_volsize.c + rm -f $@ + $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_archive -lpcp_import + check_pmiend_fdleak: check_pmiend_fdleak.c rm -f $@ - $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_import + $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_archive -lpcp_import $(LINKER_MAKERULE) check_pmi_errconv: check_pmi_errconv.c rm -f $@ - $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_import + $(CCF) $(CDEFS) -o $@ $@.c $(LDLIBS) -lpcp_archive -lpcp_import $(LINKER_MAKERULE) # --- need libpcp_web diff --git a/qa/src/GNUmakefile b/qa/src/GNUmakefile index 663c5352dd4..0bfe182ffb1 100644 --- a/qa/src/GNUmakefile +++ b/qa/src/GNUmakefile @@ -14,6 +14,7 @@ LLDFLAGS = \ -L$(TOPDIR)/src/libpcp_mmv/$(LIBPCP_ABIDIR) \ -L$(TOPDIR)/src/libpcp_web/$(LIBPCP_ABIDIR) \ -L$(TOPDIR)/src/libpcp_trace/$(LIBPCP_ABIDIR) \ + -L$(TOPDIR)/src/libpcp_archive/$(LIBPCP_ABIDIR) \ -L$(TOPDIR)/src/libpcp_import/$(LIBPCP_ABIDIR) NVIDIACFLAGS = -I$(TOPDIR)/src/pmdas/nvidia diff --git a/qa/src/check_import.c b/qa/src/check_import.c index e86a675017a..79820238d37 100644 --- a/qa/src/check_import.c +++ b/qa/src/check_import.c @@ -27,6 +27,8 @@ main(int argc, char **argv) int ctx2; int hdl1; int hdl2; + int hdl3; + pmAtomValue av; int errflag = 0; int c; int Vflag = 0; @@ -139,12 +141,17 @@ main(int argc, char **argv) hdl1 = pmiGetHandle("my.metric.foo", ""); check(hdl1, "pmiGetHandle"); + hdl3 = pmiGetHandle("my.metric.long", ""); + check(hdl3, "pmiGetHandle"); sts = pmiGetHandle("my.bad", ""); check(sts, "pmiGetHandle"); sts = pmiPutValueHandle(hdl1, "321"); check(sts, "pmiPutValueHandle"); sts = pmiPutValueHandle(0, "error"); check(sts, "pmiPutValueHandle"); + av.ul = 0; + sts = pmiPutAtomValueHandle(0, &av); + check(sts, "pmiPutAtomValueHandle"); sts = pmiPutText(PM_TEXT_PMID, PM_TEXT_ONELINE, pmID_build(245,0,1), "One line text for my.metric.foo"); @@ -333,6 +340,17 @@ main(int argc, char **argv) check(sts, "pmiWrite"); sts = pmiWrite(-1, -1); check(sts, "pmiWrite"); + av.ul = 55555; + sts = pmiPutAtomValueHandle(hdl1, &av); /* U32: PM_VAL_INSITU path */ + check(sts, "pmiPutAtomValueHandle"); + av.ll = 1234567890123LL; + sts = pmiPutAtomValueHandle(hdl3, &av); /* 64-bit: PM_VAL_DPTR path */ + check(sts, "pmiPutAtomValueHandle"); + av.ul = 0; + sts = pmiPutAtomValueHandle(hdl1, &av); /* duplicate: PMI_ERR_DUPVALUE */ + check(sts, "pmiPutAtomValueHandle"); + sts = pmiWrite(-1, -1); + check(sts, "pmiWrite"); sts = pmiPutMark(); check(sts, "pmiPutMark"); diff --git a/qa/src/check_import_append.c b/qa/src/check_import_append.c new file mode 100644 index 00000000000..0e80f4c1665 --- /dev/null +++ b/qa/src/check_import_append.c @@ -0,0 +1,132 @@ +/* + * Exercise PMI_APPEND flag to pmiStart() - create an archive then + * append additional records to it in a separate context. + * + * Copyright (c) 2026 Red Hat. All Rights Reserved. + */ + +#include +#include + +static void +check(int sts, char *name) +{ + if (sts < 0) + fprintf(stderr, "%s: Error: %s\n", name, pmiErrStr(sts)); + else { + fprintf(stderr, "%s: OK", name); + if (sts != 0) fprintf(stderr, " ->%d", sts); + fputc('\n', stderr); + } +} + +int +main(int argc, char **argv) +{ + int sts; + int ctx; + + pmSetProgname(argv[0]); + + /* + * Phase 1: create a fresh archive with two records. + */ + fprintf(stderr, "=== Phase 1: create ===\n"); + + ctx = pmiStart("appendtest", 0); + check(ctx, "pmiStart"); + + sts = pmiSetHostname("testhost.example.com"); + check(sts, "pmiSetHostname"); + + sts = pmiSetTimezone("UTC"); + check(sts, "pmiSetTimezone"); + + sts = pmiAddMetric("qa.append.counter", + PM_ID_NULL, PM_TYPE_U32, PM_INDOM_NULL, + PM_SEM_COUNTER, pmiUnits(0, 0, 0, 0, 0, 0)); + check(sts, "pmiAddMetric counter"); + + sts = pmiAddMetric("qa.append.instant", + PM_ID_NULL, PM_TYPE_DOUBLE, PM_INDOM_NULL, + PM_SEM_INSTANT, pmiUnits(0, 0, 0, 0, 0, 0)); + check(sts, "pmiAddMetric instant"); + + sts = pmiPutValue("qa.append.counter", NULL, "10"); + check(sts, "pmiPutValue counter (1)"); + sts = pmiPutValue("qa.append.instant", NULL, "1.5"); + check(sts, "pmiPutValue instant (1)"); + sts = pmiWrite(1000, 0); + check(sts, "pmiWrite (1)"); + + sts = pmiPutValue("qa.append.counter", NULL, "20"); + check(sts, "pmiPutValue counter (2)"); + sts = pmiPutValue("qa.append.instant", NULL, "2.5"); + check(sts, "pmiPutValue instant (2)"); + sts = pmiWrite(2000, 0); + check(sts, "pmiWrite (2)"); + + sts = pmiEnd(); + check(sts, "pmiEnd"); + + /* + * Phase 2: reopen with PMI_APPEND and add two more records. + * Timestamps must be strictly after the last one written (2000s). + */ + fprintf(stderr, "\n=== Phase 2: append ===\n"); + + ctx = pmiStart("appendtest", PMI_APPEND); + check(ctx, "pmiStart (PMI_APPEND)"); + + /* Metrics must be re-registered so pmiPutValue() can look them up */ + sts = pmiAddMetric("qa.append.counter", + PM_ID_NULL, PM_TYPE_U32, PM_INDOM_NULL, + PM_SEM_COUNTER, pmiUnits(0, 0, 0, 0, 0, 0)); + check(sts, "pmiAddMetric counter (append)"); + + sts = pmiAddMetric("qa.append.instant", + PM_ID_NULL, PM_TYPE_DOUBLE, PM_INDOM_NULL, + PM_SEM_INSTANT, pmiUnits(0, 0, 0, 0, 0, 0)); + check(sts, "pmiAddMetric instant (append)"); + + sts = pmiPutValue("qa.append.counter", NULL, "30"); + check(sts, "pmiPutValue counter (3)"); + sts = pmiPutValue("qa.append.instant", NULL, "3.5"); + check(sts, "pmiPutValue instant (3)"); + sts = pmiWrite(3000, 0); + check(sts, "pmiWrite (3)"); + + sts = pmiPutValue("qa.append.counter", NULL, "40"); + check(sts, "pmiPutValue counter (4)"); + sts = pmiPutValue("qa.append.instant", NULL, "4.5"); + check(sts, "pmiPutValue instant (4)"); + sts = pmiWrite(4000, 0); + check(sts, "pmiWrite (4)"); + + sts = pmiEnd(); + check(sts, "pmiEnd (append)"); + + /* + * Phase 3: PMI_APPEND on a non-existent archive should silently + * create a new one (fallback behaviour). + */ + fprintf(stderr, "\n=== Phase 3: append-creates-new ===\n"); + + ctx = pmiStart("appendnew", PMI_APPEND); + check(ctx, "pmiStart (PMI_APPEND, new archive)"); + + sts = pmiAddMetric("qa.append.counter", + PM_ID_NULL, PM_TYPE_U32, PM_INDOM_NULL, + PM_SEM_COUNTER, pmiUnits(0, 0, 0, 0, 0, 0)); + check(sts, "pmiAddMetric counter (new)"); + + sts = pmiPutValue("qa.append.counter", NULL, "1"); + check(sts, "pmiPutValue counter (new)"); + sts = pmiWrite(1000, 0); + check(sts, "pmiWrite (new)"); + + sts = pmiEnd(); + check(sts, "pmiEnd (new)"); + + exit(0); +} diff --git a/qa/src/check_volsize.c b/qa/src/check_volsize.c new file mode 100644 index 00000000000..6600afcf2e6 --- /dev/null +++ b/qa/src/check_volsize.c @@ -0,0 +1,148 @@ +/* + * Exercise pmiSetVolumeSize() - write enough records to trigger at least + * one volume rotation, verify the callback fires with the correct path, + * and confirm that pmiSetVolumeSize rejects a threshold at or below the + * archive label size. + * + * The test is written to produce deterministic pass/fail output regardless + * of the exact rotation count (which depends on internal record sizes that + * may vary between PCP versions). pmlogcheck in the QA shell script is + * the authoritative check for multi-volume archive integrity. + * + * Copyright (c) 2026 Red Hat. All Rights Reserved. + */ + +#include +#include +#include +#include +#include +#include + +static int rotate_count; +static char first_rotated[MAXPATHLEN]; + +static void +on_rotate(const char *vol_path) +{ + rotate_count++; + if (rotate_count == 1) + strncpy(first_rotated, vol_path, sizeof(first_rotated) - 1); + /* callbacks are not printed — rotation count is non-deterministic */ +} + +static void +check(int sts, const char *name) +{ + if (sts < 0) + fprintf(stderr, "%s: Error: %s\n", name, pmiErrStr(sts)); + else { + fprintf(stderr, "%s: OK", name); + if (sts != 0) fprintf(stderr, " ->%d", sts); + fputc('\n', stderr); + } +} + +int +main(int argc, char **argv) +{ + int sts; + int ctx; + int i; + int handle; + struct stat st; + /* + * Threshold must exceed the archive label size (v2=124, v3=800). + * 1500 bytes produces several rotations across 50 writes of a u32 + * counter without being so small that every write rotates. + */ + size_t threshold = 1500; + + pmSetProgname(argv[0]); + + /* === Phase 1: basic volume rotation === */ + ctx = pmiStart("volsizetest", 0); + check(ctx, "pmiStart"); + + sts = pmiSetHostname("testhost.example.com"); + check(sts, "pmiSetHostname"); + + sts = pmiSetTimezone("UTC"); + check(sts, "pmiSetTimezone"); + + sts = pmiAddMetric("qa.volsize.counter", + PM_ID_NULL, PM_TYPE_U32, PM_INDOM_NULL, + PM_SEM_COUNTER, pmiUnits(0, 0, 0, 0, 0, 0)); + check(sts, "pmiAddMetric"); + + handle = pmiGetHandle("qa.volsize.counter", NULL); + check(handle, "pmiGetHandle"); + + sts = pmiSetVolumeSize(threshold, on_rotate); + check(sts, "pmiSetVolumeSize"); + + for (i = 1; i <= 50; i++) { + pmAtomValue av; + + av.ul = (unsigned int)i; + sts = pmiPutAtomValueHandle(handle, &av); + if (sts < 0) { check(sts, "pmiPutAtomValueHandle"); break; } + sts = pmiHighResWrite((int64_t)i * 10, 0); + if (sts < 0) { check(sts, "pmiHighResWrite"); break; } + } + check(0, "pmiHighResWrite loop"); + + sts = pmiEnd(); + check(sts, "pmiEnd"); + + if (rotate_count == 0) { + fprintf(stderr, "FAIL: no volume rotation triggered\n"); + exit(1); + } + fprintf(stderr, "volume rotation: OK\n"); + + if (stat("volsizetest.0", &st) != 0) { + fprintf(stderr, "FAIL: volsizetest.0 not found\n"); + exit(1); + } + fprintf(stderr, "volsizetest.0: present\n"); + + if (stat("volsizetest.1", &st) != 0) { + fprintf(stderr, "FAIL: volsizetest.1 not found\n"); + exit(1); + } + fprintf(stderr, "volsizetest.1: present\n"); + + /* Each completed volume must not grossly exceed the threshold */ + if ((size_t)st.st_size > threshold * 3) { + fprintf(stderr, "FAIL: volume size %lld far exceeds threshold %zu\n", + (long long)st.st_size, threshold); + exit(1); + } + fprintf(stderr, "volume size bound: OK\n"); + + if (strstr(first_rotated, "volsizetest.0") == NULL) { + fprintf(stderr, "FAIL: first callback path '%s' missing 'volsizetest.0'\n", + first_rotated); + exit(1); + } + fprintf(stderr, "callback path: OK\n"); + + /* === Phase 2: label-size guard — threshold at/below label must fail === */ + ctx = pmiStart("guardtest", 0); + if (ctx < 0) { check(ctx, "pmiStart (guard)"); exit(1); } + + sts = pmiSetVolumeSize(1, on_rotate); + if (sts >= 0) { + fprintf(stderr, "FAIL: pmiSetVolumeSize(1) should have failed\n"); + exit(1); + } + fprintf(stderr, "label size guard: OK\n"); + + sts = pmiSetVolumeSize(0, NULL); + check(sts, "pmiSetVolumeSize(0) disable"); + + pmiEnd(); + + exit(0); +} diff --git a/qa/src/test_pmi.python b/qa/src/test_pmi.python index cf76af03972..7cd16ac591d 100755 --- a/qa/src/test_pmi.python +++ b/qa/src/test_pmi.python @@ -52,6 +52,12 @@ def test_pmi(self, path = OUTFILE, inherit = 0): print("pmiAddMetric: qa.one") self.assertTrue(code >= 0) + # exercise pmiWrite(0) - sec=0 with no usec previously raised TypeError + code = log.pmiPutValue("qa.one", "", "1") + print("pmiPutValue: qa.one (epoch)") + self.assertTrue(code >= 0) + log.pmiWrite(0) + # give it a value code = log.pmiPutValue("qa.one", "", "42") print("pmiPutValue: qa.one") diff --git a/src/GNUmakefile b/src/GNUmakefile index 16a71792638..c60557cef50 100644 --- a/src/GNUmakefile +++ b/src/GNUmakefile @@ -20,14 +20,16 @@ include $(TOPDIR)/src/include/builddefs INCLUDE_SUBDIR = include PMNS_SUBDIR = pmns LIBPCP_SUBDIR = libpcp libpcp_static +LIBPCP_ARCHIVE_SUBDIR = libpcp_archive +LIBPCP_IMPORT_SUBDIR = libpcp_import LIBS_SUBDIRS = \ libpcp_pmda \ libpcp_trace \ libpcp_pmcd \ libpcp_gui \ libpcp_mmv \ - libpcp_import \ libpcp_archive \ + libpcp_import \ libpcp_qed \ libpcp_qmc \ libpcp_qwt \ @@ -171,4 +173,5 @@ install_pcp : $(SUBDIRS) $(LIBPCP_SUBDIR): $(INCLUDE_SUBDIR) $(PMNS_SUBDIR): $(LIBPCP_SUBDIR) $(LIBS_SUBDIRS): $(PMNS_SUBDIR) +$(LIBPCP_IMPORT_SUBDIR): $(LIBPCP_ARCHIVE_SUBDIR) $(OTHER_SUBDIRS): $(LIBS_SUBDIRS) diff --git a/src/collectl2pcp/GNUmakefile b/src/collectl2pcp/GNUmakefile index 616b74fc5c7..4064f4a20a2 100644 --- a/src/collectl2pcp/GNUmakefile +++ b/src/collectl2pcp/GNUmakefile @@ -19,7 +19,7 @@ CFILES = collectl2pcp.c cpu.c disk.c net.c load.c timestamp.c util.c \ metrics.c header.c generic.c proc.c CMDTARGET = collectl2pcp$(EXECSUFFIX) -LLDLIBS = -L$(TOPDIR)/src/libpcp_import/src -lpcp_import $(PCPLIB) +LLDLIBS = -L$(TOPDIR)/src/libpcp_archive/src -lpcp_archive -L$(TOPDIR)/src/libpcp_import/src -lpcp_import $(PCPLIB) # LCFLAGS += -pg # diff --git a/src/include/builddefs.in b/src/include/builddefs.in index e1487be0254..2d7a8f19b0c 100644 --- a/src/include/builddefs.in +++ b/src/include/builddefs.in @@ -718,7 +718,7 @@ endif PYTHON_PREFIX=@python_prefix@ SETUP_PY_BUILD_OPTIONS = -DENABLE_PYTHON3 SETUP_PY_BUILD_OPTIONS += --include-dirs=$(TOPDIR)/src/include:$(TOPDIR)/src/include/pcp$(EXTRA_PY_INCLUDES) -SETUP_PY_BUILD_OPTIONS += --library-dirs=$(TOPDIR)/src/libpcp/src:$(TOPDIR)/src/libpcp_pmda/src:$(TOPDIR)/src/libpcp_gui/src:$(TOPDIR)/src/libpcp_import/src:$(TOPDIR)/src/libpcp_mmv/src +SETUP_PY_BUILD_OPTIONS += --library-dirs=$(TOPDIR)/src/libpcp/src:$(TOPDIR)/src/libpcp_pmda/src:$(TOPDIR)/src/libpcp_gui/src:$(TOPDIR)/src/libpcp_archive/src:$(TOPDIR)/src/libpcp_import/src:$(TOPDIR)/src/libpcp_mmv/src SETUP_PY_INSTALL_OPTIONS = --skip-build SETUP_PY_INSTALL_OPTIONS += --root="$${DIST_ROOT:-/}" ifeq "$(PYTHON_PREFIX)" "/usr" diff --git a/src/include/pcp/import.h b/src/include/pcp/import.h index 06ddb2657bf..742494cbf76 100644 --- a/src/include/pcp/import.h +++ b/src/include/pcp/import.h @@ -29,20 +29,30 @@ extern "C" { # endif #endif +/* Flags for the second argument to pmiStart() */ +#define PMI_INHERIT 0x1 /* inherit metric definitions from previous context */ +#define PMI_APPEND 0x2 /* append to an existing archive rather than creating a new one */ + /* core libpcp_import API routines */ PMI_CALL extern int pmiStart(const char *, int); PMI_CALL extern int pmiUseContext(int); PMI_CALL extern int pmiEnd(void); PMI_CALL extern int pmiSetHostname(const char *); PMI_CALL extern int pmiSetTimezone(const char *); +PMI_CALL extern int pmiSetZoneinfo(const char *); PMI_CALL extern int pmiSetVersion(int); +PMI_CALL extern int pmiSetVolumeSize(size_t, void (*)(const char *)); PMI_CALL extern int pmiAddMetric(const char *, pmID, int, pmInDom, int, pmUnits); PMI_CALL extern int pmiAddInstance(pmInDom, const char *, int); PMI_CALL extern int pmiPutValue(const char *, const char *, const char *); PMI_CALL extern int pmiGetHandle(const char *, const char *); PMI_CALL extern int pmiPutValueHandle(int, const char *); -PMI_CALL extern int pmiWrite(int, int); +PMI_CALL extern int pmiPutAtomValueHandle(int, pmAtomValue *); +PMI_CALL extern int pmiWrite(int, int); /* deprecated: not Y2038-safe, use either pmiWrite2 or pmiHighResWrite */ +PMI_CALL extern int pmiWrite2(int64_t, int); /* Y2038-safe, microsecond resolution */ +PMI_CALL extern int pmiHighResWrite(int64_t, int); /* Y2038-safe, nanosecond resolution */ PMI_CALL extern int pmiPutResult(const pmResult_v2 *); +PMI_CALL extern int pmiPutHighResResult(const pmResult *); PMI_CALL extern int pmiPutMark(void); PMI_CALL extern int pmiPutText(unsigned int, unsigned int, unsigned int, const char *); PMI_CALL extern int pmiPutLabel(unsigned int, unsigned int, unsigned int, const char *, const char *); diff --git a/src/include/pcp/libpcp.h b/src/include/pcp/libpcp.h index 18a15bba251..c5083da01f7 100644 --- a/src/include/pcp/libpcp.h +++ b/src/include/pcp/libpcp.h @@ -961,6 +961,7 @@ PCP_CALL extern int __pmLogEncodeLabel(const __pmLogLabel *, void **, size_t *); PCP_CALL extern int __pmLogChkLabel(__pmArchCtl *, __pmFILE *, __pmLogLabel *, int); PCP_CALL extern int __pmLogCreateLabel(const char *, int, __pmLogCtl *); PCP_CALL extern int __pmLogCreate(const char *, const char *, int, __pmArchCtl *, int); +PCP_CALL extern int __pmLogOpenAppend(const char *, __pmArchCtl *); PCP_CALL extern __pmFILE *__pmLogNewFile(const char *, int); PCP_CALL extern void __pmLogClose(__pmArchCtl *); PCP_CALL extern int __pmLogPutDesc(__pmArchCtl *, const pmDesc *, int, char **); diff --git a/src/libpcp/src/exports.in b/src/libpcp/src/exports.in index be0e456241a..7b82e972243 100644 --- a/src/libpcp/src/exports.in +++ b/src/libpcp/src/exports.in @@ -730,6 +730,7 @@ PCP_4.1 { PCP_4.2 { global: + __pmLogOpenAppend; #if ! HAVE_CFMAKERAW cfmakeraw; #endif diff --git a/src/libpcp/src/logutil.c b/src/libpcp/src/logutil.c index fc77bb353fb..92922ef477b 100644 --- a/src/libpcp/src/logutil.c +++ b/src/libpcp/src/logutil.c @@ -663,6 +663,130 @@ logFreeMeta(__pmLogCtl *lcp) logFreeHashText(&lcp->hashtext); } +/* + * Open an existing PCP archive for appending additional records. + * + * Volume discovery uses __pmLogFindOpen (canonical directory scan via + * readdir + __pmLogAddVolume) which correctly sets lcp->minvol and + * lcp->maxvol. The resulting read-mode handles are used to validate + * labels and load the temporal index, then closed and reopened in r+ + * mode for subsequent appending: + * + * .meta - r+ read label + seek to end; metadata appended at end + * .index - r+ read temporal index for lcp->endtime + seek to end + * .N - r+ validate volume label + seek to end-of-file + * + * acp->ac_flags is set to PM_CTXFLAG_LAST_VOLUME following the + * __pmLogOpen convention to signal we are at the most recent volume. + * + * On success: lcp->label is populated; lcp->endtime holds the last + * timestamp already in the archive; all three file handles are at + * end-of-file ready for writing; lcp->state == PM_LOG_STATE_INIT. + * + * Returns 0 on success, negative PCP error code on failure. + */ +int +__pmLogOpenAppend(const char *base, __pmArchCtl *acp) +{ + __pmLogCtl *lcp = acp->ac_log; + __pmLogLabel label = {0}; + char fname[MAXPATHLEN]; + int sts; + + /* Initialise write-path hash tables and file handles (mirrors __pmLogCreate) */ + lcp->hashpmid.nodes = lcp->hashpmid.hsize = 0; + lcp->hashindom.nodes = lcp->hashindom.hsize = 0; + lcp->trimindom.nodes = lcp->trimindom.hsize = 0; + lcp->hashlabels.nodes = lcp->hashlabels.hsize = 0; + lcp->hashtext.nodes = lcp->hashtext.hsize = 0; + lcp->tifp = lcp->mdfp = acp->ac_mfp = NULL; + lcp->last_ti.sec = -1; + lcp->last_ti.nsec = -1; + + /* + * Use the canonical directory scan to discover all data volumes, + * setting lcp->name, lcp->minvol and lcp->maxvol correctly via + * __pmLogAddVolume. Also opens .meta and .index in read mode. + */ + if ((sts = __pmLogFindOpen(acp, base)) < 0) + return sts; + + /* Read and validate the .meta archive label */ + if ((sts = __pmLogChkLabel(acp, lcp->mdfp, &label, PM_LOG_VOL_META)) < 0) { + __pmLogFreeLabel(&label); + goto fail; + } + lcp->label = label; + + /* Validate the .index label before trusting its contents */ + if ((sts = __pmLogChkLabel(acp, lcp->tifp, &label, PM_LOG_VOL_TI)) < 0) { + __pmLogFreeLabel(&label); + goto fail; + } + __pmLogFreeLabel(&label); + + /* Load temporal index: populates lcp->endtime (last timestamp) */ + if ((sts = __pmLogLoadIndex(lcp)) < 0) + goto fail; + + /* Done with the read-mode handles from __pmLogFindOpen */ + __pmFclose(lcp->mdfp); + __pmFclose(lcp->tifp); + lcp->mdfp = lcp->tifp = NULL; + + /* Free the in-memory index array - not used by the write path */ + free(lcp->ti); + lcp->ti = NULL; + lcp->numti = 0; + + /* Reopen .meta in r+ - label already validated; position at end */ + __pmLogName_r(base, PM_LOG_VOL_META, fname, sizeof(fname)); + if ((lcp->mdfp = __pmFopen(fname, "r+")) == NULL) { + sts = -oserror(); + goto fail; + } + __pmSetvbuf(lcp->mdfp, NULL, _IONBF, 0); + __pmFseek(lcp->mdfp, 0, SEEK_END); + + /* Reopen .index in r+; position at end */ + __pmLogName_r(base, PM_LOG_VOL_TI, fname, sizeof(fname)); + if ((lcp->tifp = __pmFopen(fname, "r+")) == NULL) { + sts = -oserror(); + goto fail; + } + __pmSetvbuf(lcp->tifp, NULL, _IONBF, 0); + __pmFseek(lcp->tifp, 0, SEEK_END); + + /* + * Open the highest-numbered data volume (lcp->maxvol, set by + * __pmLogFindOpen) in r+ mode, validate its label for consistency + * with the archive, then position at end-of-file for appending. + * Set PM_CTXFLAG_LAST_VOLUME following the __pmLogOpen convention. + */ + __pmLogName_r(base, lcp->maxvol, fname, sizeof(fname)); + if ((acp->ac_mfp = __pmFopen(fname, "r+")) == NULL) { + sts = -oserror(); + goto fail; + } + __pmSetvbuf(acp->ac_mfp, NULL, _IONBF, 0); + + if ((sts = __pmLogChkLabel(acp, acp->ac_mfp, &label, lcp->maxvol)) < 0) { + __pmLogFreeLabel(&label); + goto fail; + } + __pmFseek(acp->ac_mfp, 0, SEEK_END); + + acp->ac_curvol = lcp->maxvol; + acp->ac_flags |= PM_CTXFLAG_LAST_VOLUME; + lcp->state = PM_LOG_STATE_INIT; + return 0; + +fail: + __pmLogClose(acp); + logFreeMeta(lcp); + return sts; +} + /* * Close the log files. * Free up the space used by __pmLogCtl. diff --git a/src/libpcp_import/src/GNUmakefile b/src/libpcp_import/src/GNUmakefile index 77711c4d610..68e1ebb7be2 100644 --- a/src/libpcp_import/src/GNUmakefile +++ b/src/libpcp_import/src/GNUmakefile @@ -41,7 +41,7 @@ SYMTARGET = endif LCFLAGS = -DPMI_INTERNAL -LLDLIBS = -lpcp +LLDLIBS = -lpcp -lpcp_archive LDIRT = $(SYMTARGET) domain.h $(LIBCONFIG) DOMAIN = PMI_DOMAIN diff --git a/src/libpcp_import/src/archive.c b/src/libpcp_import/src/archive.c index 78b40f6ae9f..491d913b01d 100644 --- a/src/libpcp_import/src/archive.c +++ b/src/libpcp_import/src/archive.c @@ -17,9 +17,74 @@ #include "libpcp.h" #include "import.h" #include "private.h" +#include "archive.h" static __pmTimestamp stamp; +/* + * Transition a CONTEXT_APPEND context to CONTEXT_ACTIVE by delegating + * all archive-format knowledge to __pmLogOpenAppend() in libpcp. + * After return the file handles in logctl/archctl are open and positioned + * at end-of-file; the caller's pmi_context fields are updated from the + * restored archive label and last timestamp. + */ +static int +check_context_append(pmi_context *current) +{ + __pmLogCtl *lcp = ¤t->logctl; + __pmArchCtl *acp = ¤t->archctl; + int sts; + + sts = __pmLogOpenAppend(current->archive, acp); + if (sts < 0) + return sts; + + /* + * Load all existing metric and instance-domain descriptors from .meta + * into the hash tables (hashpmid, hashindom). check_metric() will use + * __pmLogLookupDesc() to skip re-writing descriptors that are already + * present, preventing .meta from growing on every timer-driven invocation + * of a short-lived collector such as sadc. + * + * __pmLogOpenAppend() left mdfp positioned at end-of-file; rewind to the + * start so __pmLogLoadMeta() can read from the beginning of the file, then + * seek back to the end ready for subsequent appended writes. + */ + __pmFseek(lcp->mdfp, 0, SEEK_SET); + if ((sts = __pmLogLoadMeta(acp)) < 0) { + __pmLogClose(acp); + return sts; + } + __pmFseek(lcp->mdfp, 0, SEEK_END); + + /* Restore version from the archive label (may differ from the default) */ + current->version = __pmLogVersion(lcp); + + /* Use the archive's hostname/timezone/zoneinfo unless the caller overrode them */ + if (current->hostname == NULL && lcp->label.hostname != NULL) + current->hostname = strdup(lcp->label.hostname); + if (current->timezone == NULL && lcp->label.timezone != NULL) + current->timezone = strdup(lcp->label.timezone); + if (current->zoneinfo == NULL && lcp->label.zoneinfo != NULL) + current->zoneinfo = strdup(lcp->label.zoneinfo); + + /* + * Seed last_stamp from the archive's end time so that the monotonicity + * check in _pmi_write() rejects out-of-order timestamps if a caller + * tries to write before the archive's existing last timestamp. + * + * Do NOT update 'stamp' here: _pmi_put_result() has already set it + * from result->timestamp (the correct current-write time) before + * calling check_context_start(). Overwriting stamp with lcp->endtime + * (the previous session's last timestamp) would produce a stale + * temporal index entry for the first record of this append session. + */ + current->last_stamp = lcp->endtime; + + current->state = CONTEXT_ACTIVE; + return 0; +} + static int check_context_start(pmi_context *current) { @@ -29,6 +94,18 @@ check_context_start(pmi_context *current) __pmArchCtl *acp; int sts; + if (current->state == CONTEXT_APPEND) { + sts = check_context_append(current); + if (sts != -ENOENT) + return sts; + /* + * Archive doesn't exist yet: fall through to create it normally. + * This makes PMI_APPEND safe to use unconditionally on first invocation + * of a timer-driven collector (e.g. sadc) before any archive exists. + */ + current->state = CONTEXT_START; + } + if (current->state != CONTEXT_START) return 0; /* ok */ @@ -49,8 +126,10 @@ check_context_start(pmi_context *current) if (current->timezone != NULL) { free(lcp->label.timezone); lcp->label.timezone = strdup(current->timezone); + } + if (current->zoneinfo != NULL) { free(lcp->label.zoneinfo); - lcp->label.zoneinfo = NULL; + lcp->label.zoneinfo = strdup(current->zoneinfo); } pmNewZone(lcp->label.timezone); current->state = CONTEXT_ACTIVE; @@ -77,11 +156,16 @@ check_context_start(pmi_context *current) static int check_indom(pmi_context *current, pmInDom indom, int *needti) { - int i; + int i, n; int sts = 0; __pmArchCtl *acp = ¤t->archctl; int type = current->version == PM_LOG_VERS03 ? TYPE_INDOM : TYPE_INDOM_V2; - __pmLogInDom lid; + __pmLogInDom lid; + __pmLogInDom old; + __pmLogInDom new_delta; + int *instlist; + char **namelist; + int needindom; for (i = 0; i < current->nindom; i++) { if (indom == current->indom[i].indom) { @@ -92,6 +176,46 @@ check_indom(pmi_context *current, pmInDom indom, int *needti) lid.instlist = current->indom[i].inst; lid.namelist = current->indom[i].name; lid.alloc = 0; + pmaSortInDom(&lid); + + n = __pmLogGetInDom(acp, indom, NULL, &instlist, &namelist); + if (n >= 0) { + old.numinst = n; + old.instlist = instlist; + old.namelist = namelist; + /* + * pmaDeltaInDom() returns: + * 0 = identical, skip write (stable indoms: CPUs, disks, etc.) + * 1 = changed, write full indom + * 2 = changed, delta record is smaller (v3 only) + */ + if (current->version == PM_LOG_VERS03) + needindom = pmaDeltaInDom(&old, &lid, &new_delta); + else + needindom = pmaSameInDom(&old, &lid) ? 0 : 1; + + if (needindom == 0) { + current->indom[i].meta_done = 1; + continue; + } + if (needindom == 2) { + new_delta.stamp = stamp; + new_delta.indom = indom; + if ((sts = __pmLogPutInDom(acp, TYPE_INDOM_DELTA, &new_delta)) < 0) + return sts; + /* + * Update hash with full indom so the next comparison + * starts from the correct full state, not the delta. + */ + if ((sts = __pmLogAddInDom(acp, TYPE_INDOM, &lid, NULL)) < 0) + return sts; + current->indom[i].meta_done = 1; + *needti = 1; + continue; + } + } + + /* first write, v2, or full indom change */ if ((sts = __pmLogPutInDom(acp, type, &lid)) < 0) return sts; @@ -116,12 +240,36 @@ check_metric(pmi_context *current, pmID pmid, int *needti) continue; if (current->metric[m].meta_done == 0) { char **namelist = ¤t->metric[m].name; + pmDesc existing; - if ((sts = __pmLogPutDesc(acp, ¤t->metric[m].desc, 1, namelist)) < 0) - return sts; + /* + * If the archive was opened for appending, __pmLogLoadMeta() will + * have populated hashpmid with all descriptors already on disk. + * Skip re-writing a descriptor that is already there, but first + * verify it is compatible with the registered metric — e.g. a + * package upgrade may have corrected a metric's type, semantics, + * instance domain or units, in which case the old archive cannot + * be extended with the new data. + */ + if (__pmLogLookupDesc(acp, pmid, &existing) == 0) { + if (existing.type != current->metric[m].desc.type) + return PM_ERR_LOGCHANGETYPE; + if (existing.sem != current->metric[m].desc.sem) + return PM_ERR_LOGCHANGESEM; + if (existing.indom != current->metric[m].desc.indom) + return PM_ERR_LOGCHANGEINDOM; + if (memcmp(&existing.units, ¤t->metric[m].desc.units, + sizeof(existing.units)) != 0) + return PM_ERR_LOGCHANGEUNITS; + current->metric[m].meta_done = 1; + } + else { + if ((sts = __pmLogPutDesc(acp, ¤t->metric[m].desc, 1, namelist)) < 0) + return sts; - current->metric[m].meta_done = 1; - *needti = 1; + current->metric[m].meta_done = 1; + *needti = 1; + } } if (current->metric[m].desc.indom != PM_INDOM_NULL) { if ((sts = check_indom(current, current->metric[m].desc.indom, needti)) < 0) @@ -156,7 +304,14 @@ newvolume(pmi_context *current) __pmFflush(acp->ac_mfp); } -static off_t flushsize = 100000; +/* + * Approximate byte interval between temporal index entries within a + * data volume. Smaller values improve seek performance at the cost + * of a slightly larger index; 100000 bytes is the historical default. + */ +#define PMI_FLUSH_INTERVAL ((off_t)100000) + +static off_t flushsize = PMI_FLUSH_INTERVAL; int _pmi_put_result(pmi_context *current, __pmResult *result) @@ -214,7 +369,7 @@ _pmi_put_result(pmi_context *current, __pmResult *result) off = __pmFtell(acp->ac_mfp) + ((__pmPDUHdr *)pb)->len - sizeof(__pmPDUHdr) + 2*sizeof(int); if (off >= max_logsz) { newvolume(current); - flushsize = 100000; + flushsize = PMI_FLUSH_INTERVAL; needti = 1; } @@ -230,7 +385,7 @@ _pmi_put_result(pmi_context *current, __pmResult *result) __pmLogPutIndex(acp, &stamp); /* and restore metadata seek pointer */ __pmFseek(lcp->mdfp, new_meta_offset, SEEK_SET); - flushsize = __pmFtell(acp->ac_mfp) + 100000; + flushsize = __pmFtell(acp->ac_mfp) + PMI_FLUSH_INTERVAL; } sts = current->version >= PM_LOG_VERS03 ? @@ -240,6 +395,32 @@ _pmi_put_result(pmi_context *current, __pmResult *result) if (sts < 0) return sts; + + /* + * User-API volume rotation: if pmiSetVolumeSize() was called, check + * the current data volume size after each successful write. When the + * threshold is reached, rotate to the next volume and invoke the + * caller's callback with the path of the just-closed volume so it can + * arrange compression or other post-processing. + * + * This is deliberately checked *after* the write so the completed + * volume contains a full, consistent record set. + */ + if (current->max_volume_bytes > 0 && + (size_t)__pmFtell(acp->ac_mfp) >= current->max_volume_bytes) { + int old_vol = acp->ac_curvol; + char vol_path[MAXPATHLEN]; + + newvolume(current); + flushsize = PMI_FLUSH_INTERVAL; + + if (current->on_volume_rotate) { + pmsprintf(vol_path, sizeof(vol_path), "%s.%d", + current->archive, old_vol); + current->on_volume_rotate(vol_path); + } + } + return 0; } @@ -249,6 +430,7 @@ _pmi_put_text(pmi_context *current) int sts; __pmArchCtl *acp = ¤t->archctl; pmi_text *tp; + char *existing; int t; int needti; @@ -287,6 +469,17 @@ _pmi_put_text(pmi_context *current) return sts; } + /* + * Skip re-writing text that is already present and unchanged in the + * archive (loaded into hashtext by __pmLogLoadMeta on append open). + */ + existing = NULL; + if (__pmLogLookupText(acp, tp->id, tp->type, &existing) == 0 && + strcmp(existing, tp->content) == 0) { + tp->meta_done = 1; + continue; + } + /* * Now write out the text record. * libpcp, via __pmLogPutText(), makes a copy of the storage pointed diff --git a/src/libpcp_import/src/exports b/src/libpcp_import/src/exports index 28a6b855688..0187d57a014 100644 --- a/src/libpcp_import/src/exports +++ b/src/libpcp_import/src/exports @@ -50,4 +50,7 @@ PCP_IMPORT_1.3 { PCP_IMPORT_1.4 { global: pmiExtraUnits; + pmiPutAtomValueHandle; + pmiSetVolumeSize; + pmiSetZoneinfo; } PCP_IMPORT_1.3; diff --git a/src/libpcp_import/src/import.c b/src/libpcp_import/src/import.c index 3ace054562a..81ba6b94b65 100644 --- a/src/libpcp_import/src/import.c +++ b/src/libpcp_import/src/import.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2022 Red Hat. + * Copyright (c) 2013-2022,2026 Red Hat. * Copyright (c) 2010 Ken McDonell. All Rights Reserved. * * This library is free software; you can redistribute it and/or modify it @@ -51,6 +51,9 @@ pmiDump(void) case CONTEXT_END: fprintf(f, "(end)"); break; + case CONTEXT_APPEND: + fprintf(f, "(append)"); + break; default: fprintf(f, "(BAD)"); break; @@ -316,7 +319,7 @@ pmiErrStr_r(int code, char *buf, int buflen) } int -pmiStart(const char *archive, int inherit) +pmiStart(const char *archive, int flags) { pmi_context *old_current; char *np; @@ -336,7 +339,7 @@ pmiStart(const char *archive, int inherit) old_current = &context_tab[c]; current = &context_tab[ncontext-1]; - current->state = CONTEXT_START; + current->state = (flags & PMI_APPEND) ? CONTEXT_APPEND : CONTEXT_START; current->version = archive_version; current->archive = strdup(archive); if (current->archive == NULL) { @@ -344,11 +347,23 @@ pmiStart(const char *archive, int inherit) } current->hostname = NULL; current->timezone = NULL; + current->zoneinfo = NULL; current->result = NULL; memset((void *)¤t->logctl, 0, sizeof(current->logctl)); memset((void *)¤t->archctl, 0, sizeof(current->archctl)); __pmLogWriterInit(¤t->archctl, ¤t->logctl); - if (inherit && old_current != NULL) { + /* + * PMI_APPEND and PMI_INHERIT may be used together: the inherited + * metric and indom definitions are carried into the new context, and + * when the append archive is opened on the first pmiWrite() call, + * each inherited descriptor is checked against what is already on + * disk. Compatible descriptors are silently skipped (no duplicate + * written); incompatible ones are rejected with PM_ERR_LOGCHANGE*. + * In the common case -- inheriting from a context that was writing + * to the same archive -- every inherited descriptor already exists + * and the combination is effectively a no-op beyond convenience. + */ + if ((flags & PMI_INHERIT) && old_current != NULL) { current->nmetric = old_current->nmetric; if (old_current->metric != NULL) { int m; @@ -534,6 +549,36 @@ pmiSetTimezone(const char *value) return current->last_sts; } +int +pmiSetZoneinfo(const char *value) +{ + char *detected; + + if (current == NULL) + return PM_ERR_NOCONTEXT; + + if (value == NULL) { + /* Auto-detect Olson name via __pmZoneinfo() (libpcp tz.c) */ + detected = __pmZoneinfo(); + if (detected == NULL) + return current->last_sts = 0; /* no zoneinfo available */ + value = detected; + } else { + detected = NULL; + } + + free(current->zoneinfo); + current->zoneinfo = strdup(value); + free(detected); + if (current->zoneinfo == NULL) { + pmNoMem("pmiSetZoneinfo", strlen(value)+1, PM_RECOV_ERR); + current->last_sts = -ENOMEM; + } else { + current->last_sts = 0; + } + return current->last_sts; +} + int pmiSetVersion(int version) { @@ -548,6 +593,43 @@ pmiSetVersion(int version) return current->last_sts; } +int +pmiSetVolumeSize(size_t max_bytes, void (*on_rotate)(const char *)) +{ + if (current == NULL) + return PM_ERR_NOCONTEXT; + /* + * Reject a threshold below the archive label size for the current + * archive version — a value smaller than the label would trigger + * a rotation cascade on every write since each new volume begins + * with a label that immediately exceeds the threshold. + * __pmLogLabelSize() returns the correct fixed on-disk size for + * both v2 (124 bytes) and v3 (800 bytes) archives. + */ + if (max_bytes > 0) { + /* + * Use the context's configured archive version to determine the + * on-disk label size. __pmLogLabelSize() reads the label magic + * which is not yet set on a fresh context, so derive the size + * directly from the version: v2=124, v3=800 (fixed-width structs). + * The version field defaults to PM_LOG_VERS02 after pmiStart(). + */ + size_t label_size; + label_size = (current->version >= PM_LOG_VERS03) + ? (size_t)(32 + PM_MAX_HOSTNAMELEN + + PM_MAX_TIMEZONELEN + PM_MAX_ZONEINFOLEN) + : (size_t)(20 + PM_LOG_MAXHOSTLEN + PM_TZ_MAXLEN); + if (max_bytes <= label_size) { + current->last_sts = PM_ERR_CONV; + return PM_ERR_CONV; + } + } + current->max_volume_bytes = max_bytes; + current->on_volume_rotate = on_rotate; + current->last_sts = 0; + return 0; +} + static int valid_pmns_name(const char *name) { @@ -885,6 +967,19 @@ pmiPutValueHandle(int handle, const char *value) return current->last_sts = _pmi_stuff_value(current, ¤t->handle[handle-1], value); } +int +pmiPutAtomValueHandle(int handle, pmAtomValue *atom) +{ + if (current == NULL) + return PM_ERR_NOCONTEXT; + if (handle <= 0 || handle > current->nhandle) + return current->last_sts = PMI_ERR_BADHANDLE; + if (atom == NULL) + return current->last_sts = PM_ERR_ARG; + + return current->last_sts = _pmi_stuff_atomvalue(current, ¤t->handle[handle-1], atom); +} + int pmiPutText(unsigned int type, unsigned int class, unsigned int id, const char *content) { diff --git a/src/libpcp_import/src/libpcp_import.pc.in b/src/libpcp_import/src/libpcp_import.pc.in index 64266b55b68..b260f42cf91 100644 --- a/src/libpcp_import/src/libpcp_import.pc.in +++ b/src/libpcp_import/src/libpcp_import.pc.in @@ -6,5 +6,5 @@ includedir=@INCDIR@ Name: libpcp_import Description: The Performance Metrics archive import library Version: @VERSION@ -Libs: -L${libdir} -lpcp -lpcp_import +Libs: -L${libdir} -lpcp -lpcp_archive -lpcp_import Cflags: -I${includedir} diff --git a/src/libpcp_import/src/private.h b/src/libpcp_import/src/private.h index 24566428800..00d71a0abc1 100644 --- a/src/libpcp_import/src/private.h +++ b/src/libpcp_import/src/private.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018,2021-2022 Red Hat. + * Copyright (c) 2013-2018,2021-2022,2026 Red Hat. * Copyright (c) 2010 Ken McDonell. All Rights Reserved. * * This library is free software; you can redistribute it and/or modify it @@ -56,26 +56,35 @@ typedef struct { char *archive; char *hostname; char *timezone; + char *zoneinfo; __pmLogCtl logctl; __pmArchCtl archctl; __pmResult *result; + /* + * Pair consecutive int fields to avoid 4-byte padding before each + * pointer on 64-bit platforms (private struct, no ABI constraint). + */ int nmetric; - pmi_metric *metric; int nindom; + pmi_metric *metric; pmi_indom *indom; int nhandle; - pmi_handle *handle; int ntext; + pmi_handle *handle; pmi_text *text; int nlabel; - pmi_label *label; int last_sts; + pmi_label *label; __pmTimestamp last_stamp; + /* optional volume rotation: 0 = disabled, callback may be NULL */ + size_t max_volume_bytes; + void (*on_volume_rotate)(const char *vol_path); } pmi_context; #define CONTEXT_START 1 #define CONTEXT_ACTIVE 2 #define CONTEXT_END 3 +#define CONTEXT_APPEND 4 /* open existing archive for appending */ #if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(IS_MINGW) # define _PMI_HIDDEN __attribute__ ((visibility ("hidden"))) @@ -83,6 +92,7 @@ typedef struct { # define _PMI_HIDDEN #endif +extern int _pmi_stuff_atomvalue(pmi_context *, pmi_handle *, pmAtomValue *) _PMI_HIDDEN; extern int _pmi_stuff_value(pmi_context *, pmi_handle *, const char *) _PMI_HIDDEN; extern int _pmi_put_result(pmi_context *, __pmResult *) _PMI_HIDDEN; extern int _pmi_put_text(pmi_context *) _PMI_HIDDEN; diff --git a/src/libpcp_import/src/stuff.c b/src/libpcp_import/src/stuff.c index 8a2fe83a0f5..479949f0415 100644 --- a/src/libpcp_import/src/stuff.c +++ b/src/libpcp_import/src/stuff.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2010 Ken McDonell. All Rights Reserved. - * Copyright (c) 2022 Red Hat. + * Copyright (c) 2022,2026 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -18,8 +18,13 @@ #include "import.h" #include "private.h" -int -_pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) +/* + * Shared setup: find or create the pmValueSet slot for hp's metric in + * current->result, add one pmValue slot, and return pointers to both. + * Returns 0 on success, or a negative error code. + */ +static int +_pmi_alloc_vp(pmi_context *current, pmi_handle *hp, pmValueSet **vspp, pmValue **vpp) { __pmResult *rp; int i; @@ -27,13 +32,6 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) pmValueSet *vsp; pmValue *vp; pmi_metric *mp; - char *end; - int dsize; - void *data; - __int64_t ll; - __uint64_t ull; - float f; - double d; size_t size; mp = ¤t->metric[hp->midx]; @@ -42,7 +40,7 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) /* first time - do not use __pmAllocResult due to realloc requirement */ current->result = (__pmResult *)calloc(1, sizeof(__pmResult)); if (current->result == NULL) { - pmNoMem("_pmi_stuff_value: result calloc", sizeof(__pmResult), PM_FATAL_ERR); + pmNoMem("_pmi_alloc_vp: result calloc", sizeof(__pmResult), PM_FATAL_ERR); } } rp = current->result; @@ -61,11 +59,11 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) size = sizeof(__pmResult) + (rp->numpmid-1)*sizeof(pmValueSet *); rp = current->result = (__pmResult *)realloc(current->result, size); if (current->result == NULL) { - pmNoMem("_pmi_stuff_value: result realloc", size, PM_FATAL_ERR); + pmNoMem("_pmi_alloc_vp: result realloc", size, PM_FATAL_ERR); } rp->vset[rp->numpmid-1] = (pmValueSet *)malloc(sizeof(pmValueSet)); if (rp->vset[rp->numpmid-1] == NULL) { - pmNoMem("_pmi_stuff_value: vset alloc", sizeof(pmValueSet), PM_FATAL_ERR); + pmNoMem("_pmi_alloc_vp: vset alloc", sizeof(pmValueSet), PM_FATAL_ERR); } vsp = rp->vset[rp->numpmid-1]; vsp->pmid = pmid; @@ -89,18 +87,59 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) size = sizeof(pmValueSet) + (rp->vset[i]->numval-1)*sizeof(pmValue); vsp = rp->vset[i] = (pmValueSet *)realloc(rp->vset[i], size); if (rp->vset[i] == NULL) { - pmNoMem("_pmi_stuff_value: vset realloc", size, PM_FATAL_ERR); + pmNoMem("_pmi_alloc_vp: vset realloc", size, PM_FATAL_ERR); } } vp = &vsp->vlist[vsp->numval-1]; vp->inst = hp->inst; + *vspp = vsp; + *vpp = vp; + return 0; +} + +/* Shared pmValueBlock allocation for heap-stored types (64-bit, float, string, ...). */ +static void +_pmi_alloc_valblock(pmi_metric *mp, pmValue *vp, int dsize, void *data) +{ + int need = dsize + PM_VAL_HDR_SIZE; + + /* logic copied from stuffvalue.c in libpcp */ + vp->value.pval = (pmValueBlock *)malloc(need < sizeof(pmValueBlock) ? sizeof(pmValueBlock) : need); + if (vp->value.pval == NULL) { + pmNoMem("_pmi_alloc_valblock: pmValueBlock", need < sizeof(pmValueBlock) ? sizeof(pmValueBlock) : need, PM_FATAL_ERR); + } + vp->value.pval->vlen = (int)need; + vp->value.pval->vtype = mp->desc.type; + memcpy((void *)vp->value.pval->vbuf, data, dsize); +} + +int +_pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) +{ + int sts; + pmValueSet *vsp; + pmValue *vp; + pmi_metric *mp; + char *end; + int dsize; + void *data; + __int64_t ll; + __uint64_t ull; + float f; + double d; + + mp = ¤t->metric[hp->midx]; + + if ((sts = _pmi_alloc_vp(current, hp, &vsp, &vp)) != 0) + return sts; + dsize = -1; switch (mp->desc.type) { case PM_TYPE_32: vp->value.lval = (__int32_t)strtol(value, &end, 10); if (*end != '\0') { if (vsp->numval == 1) vsp->numval = PM_ERR_CONV; - else rp->vset[i]->numval--; + else vsp->numval--; return PM_ERR_CONV; } if (vsp->numval == 1) vsp->valfmt = PM_VAL_INSITU; @@ -110,7 +149,7 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) vp->value.lval = (__uint32_t)strtoul(value, &end, 10); if (*end != '\0') { if (vsp->numval == 1) vsp->numval = PM_ERR_CONV; - else rp->vset[i]->numval--; + else vsp->numval--; return PM_ERR_CONV; } if (vsp->numval == 1) vsp->valfmt = PM_VAL_INSITU; @@ -120,7 +159,7 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) ll = strtoint64(value, &end, 10); if (*end != '\0') { if (vsp->numval == 1) vsp->numval = PM_ERR_CONV; - else rp->vset[i]->numval--; + else vsp->numval--; return PM_ERR_CONV; } dsize = sizeof(ll); @@ -132,7 +171,7 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) ull = strtouint64(value, &end, 10); if (*end != '\0') { if (vsp->numval == 1) vsp->numval = PM_ERR_CONV; - else rp->vset[i]->numval--; + else vsp->numval--; return PM_ERR_CONV; } dsize = sizeof(ull); @@ -144,7 +183,7 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) f = strtof(value, &end); if (*end != '\0') { if (vsp->numval == 1) vsp->numval = PM_ERR_CONV; - else rp->vset[i]->numval--; + else vsp->numval--; return PM_ERR_CONV; } dsize = sizeof(f); @@ -156,7 +195,7 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) d = strtod(value, &end); if (*end != '\0') { if (vsp->numval == 1) vsp->numval = PM_ERR_CONV; - else rp->vset[i]->numval--; + else vsp->numval--; return PM_ERR_CONV; } dsize = sizeof(d); @@ -172,22 +211,81 @@ _pmi_stuff_value(pmi_context *current, pmi_handle *hp, const char *value) default: if (vsp->numval == 1) vsp->numval = PM_ERR_TYPE; - else rp->vset[i]->numval--; + else vsp->numval--; return PM_ERR_TYPE; } - if (dsize != -1) { - /* logic copied from stuffvalue.c in libpcp */ - int need = dsize + PM_VAL_HDR_SIZE; + if (dsize != -1) + _pmi_alloc_valblock(mp, vp, dsize, data); - vp->value.pval = (pmValueBlock *)malloc(need < sizeof(pmValueBlock) ? sizeof(pmValueBlock) : need); - if (vp->value.pval == NULL) { - pmNoMem("_pmi_stuff_value: pmValueBlock", need < sizeof(pmValueBlock) ? sizeof(pmValueBlock) : need, PM_FATAL_ERR); - } - vp->value.pval->vlen = (int)need; - vp->value.pval->vtype = mp->desc.type; - memcpy((void *)vp->value.pval->vbuf, data, dsize); + return 0; +} + +int +_pmi_stuff_atomvalue(pmi_context *current, pmi_handle *hp, pmAtomValue *atom) +{ + int sts; + pmValueSet *vsp; + pmValue *vp; + pmi_metric *mp; + int dsize; + void *data; + + mp = ¤t->metric[hp->midx]; + + if ((sts = _pmi_alloc_vp(current, hp, &vsp, &vp)) != 0) + return sts; + + dsize = -1; + switch (mp->desc.type) { + case PM_TYPE_32: + vp->value.lval = atom->l; + if (vsp->numval == 1) vsp->valfmt = PM_VAL_INSITU; + break; + + case PM_TYPE_U32: + vp->value.lval = atom->ul; + if (vsp->numval == 1) vsp->valfmt = PM_VAL_INSITU; + break; + + case PM_TYPE_64: + dsize = sizeof(atom->ll); + data = (void *)&atom->ll; + if (vsp->numval == 1) vsp->valfmt = PM_VAL_DPTR; + break; + + case PM_TYPE_U64: + dsize = sizeof(atom->ull); + data = (void *)&atom->ull; + if (vsp->numval == 1) vsp->valfmt = PM_VAL_DPTR; + break; + + case PM_TYPE_FLOAT: + dsize = sizeof(atom->f); + data = (void *)&atom->f; + if (vsp->numval == 1) vsp->valfmt = PM_VAL_DPTR; + break; + + case PM_TYPE_DOUBLE: + dsize = sizeof(atom->d); + data = (void *)&atom->d; + if (vsp->numval == 1) vsp->valfmt = PM_VAL_DPTR; + break; + + case PM_TYPE_STRING: + dsize = strlen(atom->cp)+1; + data = (void *)atom->cp; + if (vsp->numval == 1) vsp->valfmt = PM_VAL_DPTR; + break; + + default: + if (vsp->numval == 1) vsp->numval = PM_ERR_TYPE; + else vsp->numval--; + return PM_ERR_TYPE; } + if (dsize != -1) + _pmi_alloc_valblock(mp, vp, dsize, data); + return 0; } diff --git a/src/perl/LogImport/Makefile.PL b/src/perl/LogImport/Makefile.PL index 359bbc83404..126e5a5c861 100644 --- a/src/perl/LogImport/Makefile.PL +++ b/src/perl/LogImport/Makefile.PL @@ -8,14 +8,14 @@ my $lddlflags; my $cccdlflags; if ($ENV{TARGET_OS} eq "mingw") { - $ldfrom = "-L$ENV{PCP_TOPDIR}/src/libpcp/src -L$ENV{PCP_TOPDIR}/src/libpcp_import/src -L$ENV{PCP_DIR}\\local\\bin -lpcp_import -lpcp LogImport.o", + $ldfrom = "-L$ENV{PCP_TOPDIR}/src/libpcp/src -L$ENV{PCP_TOPDIR}/src/libpcp_archive/src -L$ENV{PCP_TOPDIR}/src/libpcp_import/src -L$ENV{PCP_DIR}\\local\\bin -lpcp_import -lpcp_archive -lpcp LogImport.o", $inc = "-I$ENV{PCP_TOPDIR}/src/include/pcp -I/usr/include/pcp -I$ENV{PCP_DIR}\\include\\pcp -I$ENV{PCP_DIR}\\c\\include"; - $libs = ["-L$ENV{PCP_DIR}\\local\\bin", '-lpcp_import', '-lpcp']; + $libs = ["-L$ENV{PCP_DIR}\\local\\bin", '-lpcp_import', '-lpcp_archive', '-lpcp']; } else { $ldfrom = "LogImport.o", $inc = "-I$ENV{PCP_TOPDIR}/src/include/pcp -I/usr/include/pcp"; - $libs = ["-L$ENV{PCP_TOPDIR}/src/libpcp/src -L$ENV{PCP_TOPDIR}/src/libpcp_import/src -lpcp_import -lpcp"]; + $libs = ["-L$ENV{PCP_TOPDIR}/src/libpcp/src -L$ENV{PCP_TOPDIR}/src/libpcp_archive/src -L$ENV{PCP_TOPDIR}/src/libpcp_import/src -lpcp_import -lpcp_archive -lpcp"]; } if ($ENV{TARGET_OS} eq "darwin") { $lddlflags = "-bundle -undefined dynamic_lookup"; diff --git a/src/pmlogpaste/GNUmakefile b/src/pmlogpaste/GNUmakefile index b0052a58d56..d714d440c51 100644 --- a/src/pmlogpaste/GNUmakefile +++ b/src/pmlogpaste/GNUmakefile @@ -17,7 +17,7 @@ include $(TOPDIR)/src/include/builddefs CFILES = pmlogpaste.c CMDTARGET = pmlogpaste$(EXECSUFFIX) -LLDLIBS = -L$(TOPDIR)/src/libpcp_import/src -lpcp_import $(PCPLIB) +LLDLIBS = -L$(TOPDIR)/src/libpcp_archive/src -lpcp_archive -L$(TOPDIR)/src/libpcp_import/src -lpcp_import $(PCPLIB) default: $(CMDTARGET) diff --git a/src/python/distutils-setup.py b/src/python/distutils-setup.py index 91143a59c95..a9b8a03dfc7 100644 --- a/src/python/distutils-setup.py +++ b/src/python/distutils-setup.py @@ -32,7 +32,7 @@ Extension('cpmapi', ['pmapi.c'], libraries = ['pcp']), Extension('cpmda', ['pmda.c'], libraries = ['pcp_pmda', 'pcp']), Extension('cpmgui', ['pmgui.c'], libraries = ['pcp_gui']), - Extension('cpmi', ['pmi.c'], libraries = ['pcp_import']), + Extension('cpmi', ['pmi.c'], libraries = ['pcp_import', 'pcp_archive']), Extension('cmmv', ['mmv.c'], libraries = ['pcp_mmv']), ], platforms = [ 'Windows', 'Linux', 'FreeBSD', 'NetBSD', 'OpenBSD', 'Solaris', 'macOS', 'AIX' ], diff --git a/src/python/pcp/pmi.py b/src/python/pcp/pmi.py index 7b884c6e509..ab6173d85cd 100644 --- a/src/python/pcp/pmi.py +++ b/src/python/pcp/pmi.py @@ -55,6 +55,7 @@ from ctypes.util import find_library from datetime import datetime, timedelta, tzinfo from math import modf +from typing import Optional, Union # Performance Co-Pilot PMI library (C) LIBPCP_IMPORT = CDLL(find_library("pcp_import")) @@ -200,10 +201,10 @@ class pmiLogImport(object): ## # property read methods - def read_path(self): + def read_path(self) -> bytes: """ Property for archive path """ return self._path - def read_ctx(self): + def read_ctx(self) -> int: """ Property for log import context """ return self._ctx @@ -216,7 +217,7 @@ def read_ctx(self): ## # overloads - def __init__(self, path, inherit=0): + def __init__(self, path: Union[str, bytes], inherit: int = 0) -> None: if not isinstance(path, bytes): path = path.encode('utf-8') self._path = path # the archive path (file name) @@ -234,7 +235,7 @@ def __del__(self): ## # PMI Log Import Services - def pmiSetHostname(self, hostname): + def pmiSetHostname(self, hostname: str) -> int: """PMI - set the source host name for a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -246,7 +247,7 @@ def pmiSetHostname(self, hostname): raise pmiErr(status) return status - def pmiSetTimezone(self, timezone): + def pmiSetTimezone(self, timezone: str) -> int: """PMI - set the source timezone for a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) @@ -259,7 +260,7 @@ def pmiSetTimezone(self, timezone): raise pmiErr(status) return status - def pmiSetVersion(self, version): + def pmiSetVersion(self, version: int) -> int: """PMI - set the output archive version (2 or 3) """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) @@ -273,28 +274,29 @@ def pmiSetVersion(self, version): return status @staticmethod - def pmiID(domain, cluster, item): + def pmiID(domain: int, cluster: int, item: int) -> pmID: """PMI - construct a pmID data structure (helper routine) """ return LIBPCP_IMPORT.pmiID(domain, cluster, item) @staticmethod - def pmiCluster(domain, cluster): + def pmiCluster(domain: int, cluster: int) -> pmID: """PMI - construct a pmID data structure (helper routine) """ return LIBPCP_IMPORT.pmiCluster(domain, cluster) @staticmethod - def pmiInDom(domain, serial): + def pmiInDom(domain: int, serial: int) -> pmInDom: """PMI - construct a pmInDom data structure (helper routine) """ return LIBPCP_IMPORT.pmiInDom(domain, serial) @staticmethod - def pmiUnits(dim_space, dim_time, dim_count, - scale_space, scale_time, scale_count): + def pmiUnits(dim_space: int, dim_time: int, dim_count: int, + scale_space: int, scale_time: int, scale_count: int) -> pmUnits: """PMI - construct a pmiUnits data structure (helper routine) """ return LIBPCP_IMPORT.pmiUnits(dim_space, dim_time, dim_count, scale_space, scale_time, scale_count) - def pmiAddMetric(self, name, pmid, typed, indom, sem, units): + def pmiAddMetric(self, name: str, pmid: pmID, typed: int, + indom: pmInDom, sem: int, units: pmUnits) -> int: """PMI - add a new metric definition to a Log Import context """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -307,7 +309,7 @@ def pmiAddMetric(self, name, pmid, typed, indom, sem, units): raise pmiErr(status) return status - def pmiAddInstance(self, indom, instance, instid): + def pmiAddInstance(self, indom: pmInDom, instance: str, instid: int) -> int: """PMI - add element to an instance domain in a Log Import context """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -319,7 +321,7 @@ def pmiAddInstance(self, indom, instance, instid): raise pmiErr(status) return status - def pmiPutValue(self, name, inst, value): + def pmiPutValue(self, name: str, inst: Optional[str], value: str) -> int: """PMI - add a value for a metric-instance pair """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -339,7 +341,7 @@ def pmiPutValue(self, name, inst, value): raise pmiErr(status) return status - def pmiGetHandle(self, name, inst): + def pmiGetHandle(self, name: str, inst: Optional[str]) -> int: """PMI - define a handle for a metric-instance pair """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -356,7 +358,7 @@ def pmiGetHandle(self, name, inst): raise pmiErr(status) return status - def pmiPutValueHandle(self, handle, value): + def pmiPutValueHandle(self, handle: int, value: str) -> int: """PMI - add a value for a metric-instance pair via a handle """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -368,7 +370,7 @@ def pmiPutValueHandle(self, handle, value): raise pmiErr(status) return status - def pmiHighResWrite(self, sec, nsec): + def pmiHighResWrite(self, sec: int, nsec: int) -> int: """PMI - flush data to a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -378,12 +380,13 @@ def pmiHighResWrite(self, sec, nsec): raise pmiErr(status) return status - def pmiWrite(self, sec, usec=None): + def pmiWrite(self, sec: Union[int, float, datetime], + usec: Optional[int] = None) -> int: """PMI - flush data to a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: raise pmiErr(status) - if sec and not usec: + if usec is None: if isinstance(sec, datetime): sec = float((sec - self._epoch).total_seconds()) if isinstance(sec, float): @@ -397,7 +400,7 @@ def pmiWrite(self, sec, usec=None): raise pmiErr(status) return status - def pmiPutMark(self): + def pmiPutMark(self) -> int: """PMI - write a record to a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -407,7 +410,7 @@ def pmiPutMark(self): raise pmiErr(status) return status - def put_result(self, result): + def put_result(self, result: pmResult_v2) -> int: """PMI - add a data record to a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -417,7 +420,7 @@ def put_result(self, result): raise pmiErr(status) return status - def put_highres_result(self, result): + def put_highres_result(self, result: pmResult) -> int: """PMI - add a data record to a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -427,7 +430,7 @@ def put_highres_result(self, result): raise pmiErr(status) return status - def pmiPutText(self, typ, cls, ident, content): + def pmiPutText(self, typ: int, cls: int, ident: int, content: str) -> int: """PMI - add a text record to a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -439,7 +442,8 @@ def pmiPutText(self, typ, cls, ident, content): raise pmiErr(status) return status - def pmiPutLabel(self, typ, ident, inst, name, content): + def pmiPutLabel(self, typ: int, ident: int, inst: int, + name: str, content: str) -> int: """PMI - add a label record to a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: @@ -454,11 +458,11 @@ def pmiPutLabel(self, typ, ident, inst, name, content): return status @staticmethod - def pmiDump(): + def pmiDump() -> None: """PMI - dump the current Log Import contexts (diagnostic) """ LIBPCP_IMPORT.pmiDump() - def pmiEnd(self): + def pmiEnd(self) -> int: """PMI - close current context and finish a Log Import archive """ status = LIBPCP_IMPORT.pmiUseContext(self._ctx) if status < 0: diff --git a/src/python/setup.py b/src/python/setup.py index 0f9def6cf27..a5b0729d0b9 100644 --- a/src/python/setup.py +++ b/src/python/setup.py @@ -42,7 +42,7 @@ Extension('cpmapi', ['pmapi.c'], libraries = ['pcp']), Extension('cpmda', ['pmda.c'], libraries = ['pcp_pmda', 'pcp']), Extension('cpmgui', ['pmgui.c'], libraries = ['pcp_gui', 'pcp']), - Extension('cpmi', ['pmi.c'], libraries = ['pcp_import', 'pcp']), + Extension('cpmi', ['pmi.c'], libraries = ['pcp_import', 'pcp_archive', 'pcp']), Extension('cmmv', ['mmv.c'], libraries = ['pcp_mmv', 'pcp']), ], keywords = ['performance', 'analysis', 'monitoring' ],