.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.43)
.\"
.\" Standard preamble:
.\" ========================================================================
.de Sp \" Vertical space (when we can't use .PP)
.if t .sp .5v
.if n .sp
..
.de Vb \" Begin verbatim text
.ft CW
.nf
.ne \\$1
..
.de Ve \" End verbatim text
.ft R
.fi
..
.\" Set up some character translations and predefined strings.  \*(-- will
.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
.\" double quote, and \*(R" will give a right double quote.  \*(C+ will
.\" give a nicer C++.  Capital omega is used to do unbreakable dashes and
.\" therefore won't be available.  \*(C` and \*(C' expand to `' in nroff,
.\" nothing in troff, for use with C<>.
.tr \(*W-
.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
.ie n \{\
.    ds -- \(*W-
.    ds PI pi
.    if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
.    if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\"  diablo 12 pitch
.    ds L" ""
.    ds R" ""
.    ds C` ""
.    ds C' ""
'br\}
.el\{\
.    ds -- \|\(em\|
.    ds PI \(*p
.    ds L" ``
.    ds R" ''
.    ds C`
.    ds C'
'br\}
.\"
.\" Escape single quotes in literal strings from groff's Unicode transform.
.ie \n(.g .ds Aq \(aq
.el       .ds Aq '
.\"
.\" If the F register is >0, we'll generate index entries on stderr for
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
.\" entries marked with X<> in POD.  Of course, you'll have to process the
.\" output yourself in some meaningful fashion.
.\"
.\" Avoid warning from groff about undefined register 'F'.
.de IX
..
.nr rF 0
.if \n(.g .if rF .nr rF 1
.if (\n(rF:(\n(.g==0)) \{\
.    if \nF \{\
.        de IX
.        tm Index:\\$1\t\\n%\t"\\$2"
..
.        if !\nF==2 \{\
.            nr % 0
.            nr F 2
.        \}
.    \}
.\}
.rr rF
.\"
.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
.\" Fear.  Run.  Save yourself.  No user-serviceable parts.
.    \" fudge factors for nroff and troff
.if n \{\
.    ds #H 0
.    ds #V .8m
.    ds #F .3m
.    ds #[ \f1
.    ds #] \fP
.\}
.if t \{\
.    ds #H ((1u-(\\\\n(.fu%2u))*.13m)
.    ds #V .6m
.    ds #F 0
.    ds #[ \&
.    ds #] \&
.\}
.    \" simple accents for nroff and troff
.if n \{\
.    ds ' \&
.    ds ` \&
.    ds ^ \&
.    ds , \&
.    ds ~ ~
.    ds /
.\}
.if t \{\
.    ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
.    ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
.    ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
.    ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
.    ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
.    ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
.\}
.    \" troff and (daisy-wheel) nroff accents
.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
.ds ae a\h'-(\w'a'u*4/10)'e
.ds Ae A\h'-(\w'A'u*4/10)'E
.    \" corrections for vroff
.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
.    \" for low resolution devices (crt and lpr)
.if \n(.H>23 .if \n(.V>19 \
\{\
.    ds : e
.    ds 8 ss
.    ds o a
.    ds d- d\h'-1'\(ga
.    ds D- D\h'-1'\(hy
.    ds th \o'bp'
.    ds Th \o'LP'
.    ds ae ae
.    ds Ae AE
.\}
.rm #[ #] #H #V #F C
.\" ========================================================================
.\"
.IX Title "MAKEHISTORY 8"
.TH MAKEHISTORY 8 "2024-05-19" "INN 2.7.2" "InterNetNews Documentation"
.\" For nroff, turn off justification.  Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
.nh
.SH "NAME"
makehistory \- Initialize or rebuild INN history database
.SH "SYNOPSIS"
.IX Header "SYNOPSIS"
\&\fBmakehistory\fR [\fB\-abFIOSx\fR] [\fB\-f\fR \fIfilename\fR] [\fB\-l\fR \fIcount\fR]
[\fB\-L\fR \fIload-average\fR] [\fB\-s\fR \fIsize\fR] [\fB\-T\fR \fItmpdir\fR]
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
\&\fBmakehistory\fR rebuilds the \fBhistory\fR\|(5) text file, which contains a list of
message-IDs of articles already seen by the server.  It can also be used
to rebuild the overview database.  Note that even though the \fIdbz\fR indices
for the \fIhistory\fR file are also rebuilt by \fBmakehistory\fR, it is useful
to run \fBmakedbz\fR\|(8) after \fBmakehistory\fR\|(8) in order to improve the efficiency
of the indices (\fBmakehistory\fR does not know how large to make the hash
table at first run, unless the size is given by the \fB\-s\fR flag).
.PP
The default location of the \fIhistory\fR text file is \fIpathdb\fR/history; to
specify an alternate location, use the \fB\-f\fR flag.
.PP
By default, \fBmakehistory\fR will scan the entire spool, using the storage
manager, and write a history line for every article.  To also generate
overview information, use the \fB\-O\fR flag.
.PP
If a malformed article is found in the news spool, in a way which prevents its
integration into the history or overview data, a log line will be output and
the malformed article will just be skipped.
.SH "OVERVIEW REBUILD"
.IX Header "OVERVIEW REBUILD"
\&\fI\s-1WARNING\s0\fR: If you're trying to rebuild the overview database, be sure to
stop \fBinnd\fR\|(8) and delete or zero out the existing database before you start
for the best results.  An overview rebuild should not be done while the
server is running.  Unless the existing overview is deleted, you may end
up with problems like out-of-order overview entries, excessively large
overview buffers, and the like.
.PP
If \fIovmethod\fR in \fIinn.conf\fR is \f(CW\*(C`ovdb\*(C'\fR, you must have the ovdb processes
running while rebuilding overview.  ovdb needs them available while
writing overview entries.  You can start them by hand separate from the
rest of the server by running \fBovdb_init\fR; see \fBovdb_init\fR\|(8) for more
details.
.PP
Similarly, if \fIovmethod\fR in \fIinn.conf\fR is \f(CW\*(C`ovsqlite\*(C'\fR, you must
have the \fBovsqlite-server\fR process running while rebuilding overview.
See \fBovsqlite\-server\fR\|(8) for more details and how to start it by hand.
.PP
Rebuilding overview data is as straight-forward as:
.IP "1." 4
Checking that the configuration file of the new overview storage method is
present in \fIpathetc\fR and fits your needs (\fIbuffindexed.conf\fR, \fIovdb.conf\fR
or \fIovsqlite.conf\fR).  Note that the tradindexed overview storage method does
not have a dedicated configuration file.
.IP "2." 4
Making sure that \s-1INN\s0 is stopped (\f(CW\*(C`rc.news stop\*(C'\fR as the news user, or
whichever command you're using).
.IP "3." 4
Setting the new overview storage method in the \fIovmethod\fR parameter
in \fIinn.conf\fR.
.IP "4." 4
Making sure that the directory specified by the \fIpathoverview\fR parameter in
\&\fIinn.conf\fR exists and is empty (or contains freshly created buffindexed
buffers, if using that overview storage method).  Otherwise, rename
the current directory (to backup existing overview data) and re-create
\&\fIpathoverview\fR as the news user.
.IP "5." 4
Starting \fBovdb_init\fR or \fBovsqlite-server\fR as the news user if the new
overview storage method is respectively ovdb or ovsqlite.
.IP "6." 4
Running \f(CW\*(C`makehistory \-O \-x \-F\*(C'\fR and waiting for the command to finish.
(You may notice a few logs about articles for which overview data cannot be
inserted into the new overview storage method.  As long as there aren't tons
of them, it is normal, notably because there is an internal limit in the
length of overview data generated by \fBmakehistory\fR, contrary to \fBinnd\fR.
Unfortunately, these rare articles won't be present in the new overview.)
.IP "7." 4
Stopping ovdb or ovsqlite helper programs if you started them during the
previous steps (running \f(CW\*(C`rc.news stop\*(C'\fR as the news user will stop them;
do not mind the messages related to the fact that the news server was not
running).
.IP "8." 4
Starting \s-1INN\s0 and checking the logs to make sure everything is fine.
You will normally notice that the \fIactive\fR file is renumbered
(\fBrc.news\fR takes care of that when run after an overview rebuild;
otherwise, manually run \f(CW\*(C`ctlinnd renumber \*(Aq\*(Aq\*(C'\fR).
.SH "OPTIONS"
.IX Header "OPTIONS"
.IP "\fB\-a\fR" 4
.IX Item "-a"
Append to the \fIhistory\fR file rather than generating a new one.  If you
append to the main \fIhistory\fR file, make sure \fBinnd\fR\|(8) is throttled or not
running, or you can corrupt the history.
.IP "\fB\-b\fR" 4
.IX Item "-b"
Delete any messages found in the spool that do not have valid Message-ID
header fields in them.
.IP "\fB\-F\fR" 4
.IX Item "-F"
Fork a separate process to flush overview data to disk rather than doing
it directly.  The advantage of this is that it allows \fBmakehistory\fR to
continue to collect more data from the spool while the first batch of data
is being written to the overview database.  The disadvantage is that up to
twice as much temporary disk space will be used for the generated overview
data.  This option only makes sense in combination with \fB\-O\fR.  With
buffindexed, the \fBoverchan\fR program is invoked to write overview.
.IP "\fB\-f\fR \fIfilename\fR" 4
.IX Item "-f filename"
Rather than writing directly to \fIpathdb\fR/history, instead write to
\&\fIfilename\fR, also in \fIpathdb\fR.
.IP "\fB\-I\fR" 4
.IX Item "-I"
Don't store overview data for articles numbered lower than the lowest
article number in \fIactive\fR.  This is useful if there are for whatever
reason old articles on disk that shouldn't be available to readers or put
into the overview database.
.IP "\fB\-l\fR \fIcount\fR" 4
.IX Item "-l count"
This option specifies how many articles to process before writing the
accumulated overview information out to the overview database.  The
default is \f(CW10000\fR.  Since overview write performance is faster with
sorted data, each \*(L"batch\*(R" gets sorted.  Increasing the batch size
with this option may further improve write performance, at the cost
of longer sort times.  Also, temporary space will be needed to store
the overview batches.  At a rough estimate, about 300 * \fIcount\fR bytes
of temporary space will be required (not counting temp files created
by \fBsort\fR\|(1)).  See the description of the \fB\-T\fR option for how to
specify the temporary storage location.  This option has no effect
with buffindexed, because buffindexed does not need sorted
overview and no batching is done.
.IP "\fB\-L\fR \fIload-average\fR" 4
.IX Item "-L load-average"
Temporarily pause activities if the system load average exceeds the
specified level \fIload-average\fR.  This allows \fBmakehistory\fR to run
on a system being used for other purposes without monopolizing system
resources and thus making the response time for other applications
unacceptably slow.  Using \fBnice\fR\|(1) does not help much for that because
the problem comes from disk I/O usage, and \fBionice\fR\|(1) is not always
available or efficient.
.IP "\fB\-O\fR" 4
.IX Item "-O"
Create the overview database as well as the \fIhistory\fR file.  Overview
information is only required if the server supports readers; it is not
needed for a transit-only server (see \fIenableoverview\fR in \fBinn.conf\fR\|(5)).
If you are using the buffindexed overview storage method, erase all of
your overview buffers before running \fBmakehistory\fR with \fB\-O\fR.
.IP "\fB\-S\fR" 4
.IX Item "-S"
Rather than storing the overview data into the overview database, just write
it to standard output in a form suitable for feeding to \fBoverchan\fR later
if wished.  When this option is used, \fB\-F\fR, \fB\-I\fR, \fB\-l\fR, and \fB\-T\fR are
ignored.  This option only makes sense in combination with \fB\-O\fR.
.IP "\fB\-s\fR \fIsize\fR" 4
.IX Item "-s size"
Size the history database for approximately \fIsize\fR key-value pairs
(i.e. lines in \fIhistory\fR).  Accurately specifying the size is an optimization
that will create a more efficient database.  (The size should be the estimated
eventual number of articles, typically the size of the old \fIhistory\fR file,
in lines.)
.Sp
By default, \fBmakehistory\fR will create a database optimized for handling about
6,000,000 articles (or 500,000 if the slower tagged hash format is used).
This size does not limit the number of articles the news server can store
in its \fIhistory\fR file.  It will just get slower when that optimal size is
exceeded, until the next run of \fBnews.daily\fR which will appropriately resize
it.
.IP "\fB\-T\fR \fItmpdir\fR" 4
.IX Item "-T tmpdir"
If \fB\-O\fR is given, \fBmakehistory\fR needs a location to write temporary
overview data.  By default, it uses \fIpathtmp\fR, set in \fIinn.conf\fR, but if
this option is given, the provided \fItmpdir\fR is used instead.  This is
also used for temporary files created by \fBsort\fR\|(1) (which is invoked in the
process of writing overview information since sorted overview information
writes faster).  By default, \fBsort\fR usually uses your system temporary
directory; see the \fBsort\fR\|(1) man page on your system to be sure.
.IP "\fB\-x\fR" 4
.IX Item "-x"
If this option is given, \fBmakehistory\fR won't write out \fIhistory\fR file
entries.  This is useful mostly for building overview without generating
a new \fIhistory\fR file.
.SH "EXAMPLES"
.IX Header "EXAMPLES"
Here's a typical example of rebuilding the entire history and overview
database, removing broken articles in the news spool.  This uses the
default temporary file locations and should be done while \fBinnd\fR isn't
running (or is throttled).
.PP
.Vb 1
\&    makehistory \-b \-f history.n \-O \-l 30000 \-I
.Ve
.PP
This will rebuild the overview (if using buffindexed, erase the
existing overview buffers before running this command) and leave a new
\&\fIhistory\fR file as \f(CW\*(C`history.n\*(C'\fR in \fIpathdb\fR.  To preserve all of the history
entries from the old \fIhistory\fR file that correspond to rejected articles or
expired articles, follow the above command with:
.PP
.Vb 2
\&    cd <pathdb>
\&    awk \*(AqNF == 2 { print }\*(Aq < history >> history.n
.Ve
.PP
(replacing the path with your \fIpathdb\fR, if it isn't the default).  Then
look over the new \fIhistory\fR file for problems and run:
.PP
.Vb 1
\&    makedbz \-s \`wc \-l < history.n\` \-f history.n
.Ve
.PP
Then rename all of the files matching \f(CW\*(C`history.n.*\*(C'\fR to \f(CW\*(C`history.*\*(C'\fR,
replacing the current history database and indices.  After that, it's safe
to unthrottle \fBinnd\fR.
.PP
For a simpler example:
.PP
.Vb 1
\&    makehistory \-b \-f history.n \-I \-O
.Ve
.PP
will scan the spool, removing broken articles and generating history and
overview entries for articles missing from history.
.PP
To pre-size the \fIhistory\fR file for 100,000,000 articles, and generate
overview data at the same time, you may directly use the following command:
.PP
.Vb 1
\&    makehistory \-O \-s 100000000
.Ve
.PP
You then do not need running \fBmakedbz\fR as the \fIhistory\fR file has already
been generated and optimized for the expected number of articles.
.PP
To just rebuild overview:
.PP
.Vb 1
\&    makehistory \-O \-x \-F
.Ve
.SH "FILES"
.IX Header "FILES"
.IP "\fIpathdb\fR/history" 4
.IX Item "pathdb/history"
This is the default output file for \fBmakehistory\fR.
.IP "\fIpathtmp\fR" 4
.IX Item "pathtmp"
Where temporary files are written unless \fB\-T\fR is given.
.SH "HISTORY"
.IX Header "HISTORY"
Originally written by Rich \f(CW$alz\fR <rsalz@uunet.uu.net> for InterNetNews and
updated by various other people since.
.SH "SEE ALSO"
.IX Header "SEE ALSO"
\&\fBactive\fR\|(5), \fBctlinnd\fR\|(8), \fBhistory\fR\|(5), \fBinn.conf\fR\|(5), \fBinnd\fR\|(8), \fBlibinn_dbz\fR\|(3),
\&\fBmakedbz\fR\|(8), \fBovdb_init\fR\|(8), \fBoverchan\fR\|(8), \fBovsqlite\-server\fR\|(8).