mirror of
https://github.com/flokoe/bash-hackers-wiki.git
synced 2024-11-01 23:03:05 +01:00
97 lines
53 KiB
HTML
97 lines
53 KiB
HTML
|
<!doctype html><html lang=en class=no-js> <head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><link href=https://flokoe.github.io/bash-hackers-wiki/commands/builtin/mapfile/ rel=canonical><link href=../local/ rel=prev><link href=../printf/ rel=next><link rel=icon href=../../../assets/images/favicon.png><meta name=generator content="mkdocs-1.6.1, mkdocs-material-9.5.40"><title>The mapfile builtin command - The Bash Hackers Wiki</title><link rel=stylesheet href=../../../assets/stylesheets/main.8c3ca2c6.min.css><link rel=stylesheet href=../../../assets/stylesheets/palette.06af60db.min.css><link rel=preconnect href=https://fonts.gstatic.com crossorigin><link rel=stylesheet href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback"><style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style><script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script></head> <body dir=ltr data-md-color-scheme=default data-md-color-primary=indigo data-md-color-accent=indigo> <input class=md-toggle data-md-toggle=drawer type=checkbox id=__drawer autocomplete=off> <input class=md-toggle data-md-toggle=search type=checkbox id=__search autocomplete=off> <label class=md-overlay for=__drawer></label> <div data-md-component=skip> <a href=#the-mapfile-builtin-command class=md-skip> Skip to content </a> </div> <div data-md-component=announce> </div> <header class=md-header data-md-component=header> <nav class="md-header__inner md-grid" aria-label=Header> <a href=../../.. title="The Bash Hackers Wiki" class="md-header__button md-logo" aria-label="The Bash Hackers Wiki" data-md-component=logo> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg> </a> <label class="md-header__button md-icon" for=__drawer> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg> </label> <div class=md-header__title data-md-component=header-title> <div class=md-header__ellipsis> <div class=md-header__topic> <span class=md-ellipsis> The Bash Hackers Wiki </span> </div> <div class=md-header__topic data-md-component=header-topic> <span class=md-ellipsis> The mapfile builtin command </span> </div> </div> </div> <form class=md-header__option data-md-component=palette> <input class=md-option data-md-color-media data-md-color-scheme=default data-md-color-primary=indigo data-md-color-accent=indigo aria-label="Switch to dark mode" type=radio name=__palette id=__palette_0> <label class="md-header__button md-icon" title="Switch to dark mode" for=__palette_1 hidden> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg> </label> <input class=md-option data-md-color-media data-md-color-scheme=slate data-md-color-primary=indigo data-md-color-accent=indigo aria-label="Switch to light mode" type=radio name=__palette id=__palette_1> <label class="md-header__button md-icon" title="Switch to light mode" for=__palette_0 hidden> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg> </label> </form> <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.med
|
||
|
|
|
||
|
|
readarray [-n COUNT] [-O ORIGIN] [-s COUNT] [-t] [-u FD] [-C CALLBACK] [-c QUANTUM] [ARRAY]
|
||
|
|
</code></pre></div> <h2 id=description>Description<a class=headerlink href=#description title="Permanent link">¶</a></h2> <p>This builtin is also accessible using the command name <code>readarray</code>.</p> <p><code>mapfile</code> is one of the two builtin commands primarily intended for handling standard input (the other being <code>read</code>). <code>mapfile</code> reads lines of standard input and assigns each to the elements of an indexed array. If no array name is given, the default array name is <code>MAPFILE</code>. The target array must be a "normal" integer indexed array.</p> <p><code>mapfile</code> returns success (0) unless an invalid option is given or the given array <code>ARRAY</code> is set readonly.</p> <p>Option Description</p> <hr> <p><code>-c QUANTUM</code> Specifies the number of lines that have to be read between every call to the callback specified with <code>-C</code>. The default QUANTUM is 5000 <code>-C CALLBACK</code> Specifies a callback. The string <code>CALLBACK</code> can be any shell code, the index of the array that will be assigned, and the line is appended at evaluation time. <code>-n COUNT</code> Reads at most <code>COUNT</code> lines, then terminates. If <code>COUNT</code> is 0, then all lines are read (default). <code>-O ORIGIN</code> Starts populating the given array <code>ARRAY</code> at the index <code>ORIGIN</code> rather than clearing it and starting at index 0. <code>-s COUNT</code> Discards the first <code>COUNT</code> lines read. <code>-t</code> Remove any trailing newline from a line read, before it is assigned to an array element. <code>-u FD</code> Read from filedescriptor <code>FD</code> rather than standard input.</p> <p>While <code>mapfile</code> isn't a common or portable shell feature, it's functionality will be familiar to many programmers. Almost all programming languages (aside from shells) with support for compound datatypes like arrays, and which handle open file objects in the traditional way, have some analogous shortcut for easily reading all lines of some input as a standard feature. In Bash, <code>mapfile</code> in itself can't do anything that couldn't already be done using read and a loop, and if portability is even a slight concern, should never be used. However, it does <em>significantly</em> outperform a read loop, and can make for shorter and cleaner code - especially convenient for interactive use.</p> <h2 id=examples>Examples<a class=headerlink href=#examples title="Permanent link">¶</a></h2> <p>Here's a real-world example of interactive use borrowed from Gentoo workflow. Xorg updates require rebuilding drivers, and the Gentoo-suggested command is less than ideal, so let's Bashify it. The first command produces a list of packages, one per line. We can read those into the array named "args" using <code>mapfile</code>, stripping trailing newlines with the '<code>-t</code>' option. The resulting array is then expanded into the arguments of the <code>emerge</code> command - an interface to Gentoo's package manager. This type of usage can make for a safe and effective replacement for xargs(1) in certain situations. Unlike xargs, all arguments are guaranteed to be passed to a single invocation of the command with no wordsplitting, pathname expansion, or other monkey business.</p> <div class=highlight><pre><span></span><code># eix --only-names -IC x11-drivers | { mapfile -t args; emerge -av1 "${args[@]}" <&1; }
|
||
|
|
</code></pre></div> <p>Note the use of command grouping to keep the emerge command inside the pipe's subshell and within the scope of "args". Also note the unusual redirection. This is because the -a flag makes emerge interactive, asking the user for confirmation before continuing, and checking with isatty(3) to abort if stdin isn't pointed at a terminal. Since stdin of the entire command group is still coming from the pipe even though mapfile has read all available input, we just borrow FD 1 as it just so happens to be pointing where we want it. More on this over at greycat's wiki: <a href=http://mywiki.wooledge.org/BashFAQ/024>http://mywiki.wooledge.org/BashFAQ/024</a></p> <h3 id=the-callback>The callback<a class=headerlink href=#the-callback title="Permanent link">¶</a></h3> <p>This is one of the more unusual features of a Bash builtin. As far as I'm able to tell, the exact behavior is as follows: If defined, as each line is read, the code contained within the string argument to the -C flag is evaluated and executed <em>before</em> the assignment of each array element. There are no restrictions to this string, which can be any arbitrary code, however, two additional "words" are automatically appended to the end before evaluation: the index, and corresponding line of data to be assigned to the next array element. Since all this happens before assignment, the callback feature cannot be used to modify the element to be assigned, though it can read and modify any array elements already assigned.</p> <p>A very simple example might be to use it as a kind of progress bar. This will print a dot for each line read. Note the escaped comment to hide the appended words from printf.</p> <div class=highlight><pre><span></span><code>$ printf '%s\n' {1..5} | mapfile -c 1 -C 'printf . \#' )
|
||
|
|
.....
|
||
|
|
</code></pre></div> <p>Really, the intended usage is for the callback to just contain the name of a function, with the extra words passed to it as arguments. If you're going to use callbacks at all, this is probably the best way because it allows for easy access to the arguments with no ugly "code in a string".</p> <div class=highlight><pre><span></span><code>$ foo() { echo "|$1|"; }; mapfile -n 11 -c 2 -C 'foo' <file
|
||
|
|
|2|
|
||
|
|
|4|
|
||
|
|
etc..
|
||
|
|
</code></pre></div> <p>For the sake of completeness, here are some more complicated examples inspired by a question asked in #bash - how to prepend something to every line of some input, and then output even and odd lines to separate files. This is far from the best possible answer, but hopefully illustrates the callback behavior:</p> <div class=highlight><pre><span></span><code>$ { printf 'input%s\n' {1..10} | mapfile -c 1 -C '>&$(( (${#x[@]} % 2) + 3 )) printf -- "%.sprefix %s"' x; } 3>outfile0 4>outfile1
|
||
|
|
$ cat outfile{0,1}
|
||
|
|
prefix input1
|
||
|
|
prefix input3
|
||
|
|
prefix input5
|
||
|
|
prefix input7
|
||
|
|
prefix input9
|
||
|
|
prefix input2
|
||
|
|
prefix input4
|
||
|
|
prefix input6
|
||
|
|
prefix input8
|
||
|
|
prefix input10
|
||
|
|
</code></pre></div> <p>Since redirects are syntactically allowed anywhere in a command, we put it before the printf to stay out of the way of additional arguments. Rather than opening "outfile<n>" for appending on each call by calculating the filename, open an FD for each first and calculate which FD to send output to by measuring the size of x mod 2. The zero-width format specification is used to absorb the index number argument.</p> <p>Another variation might be to add each of these lines to the elements of separate arrays. I'll leave dissecting this one as an exercise for the reader. This is quite the hack but illustrates some interesting properties of printf -v and mapfile -C (which you should probably never use in real code).</p> <div class=highlight><pre><span></span><code>$ y=( 'odd[j]' 'even[j++]' ); printf 'input%s\n' {1..10} | { mapfile -tc 1 -C 'printf -v "${y[${#x[@]} % 2]}" -- "%.sprefix %s"' x; printf '%s\n' "${odd[@]}" '' "${even[@]}"; }
|
||
|
|
prefix input1
|
||
|
|
prefix input3
|
||
|
|
prefix input5
|
||
|
|
prefix input7
|
||
|
|
prefix input9
|
||
|
|
|
||
|
|
prefix input2
|
||
|
|
prefix input4
|
||
|
|
prefix input6
|
||
|
|
prefix input8
|
||
|
|
prefix input10
|
||
|
|
</code></pre></div> <p>This example based on yet another #bash question illustrates mapfile in combination with read. The sample input is the heredoc to <code>main</code>. The goal is to build a "struct" based upon records in the input file made up of the numbers following the colon on each line. Every 3<sup>rd</sup> line is a key followed by 2 corresponding fields. The showRecord function takes a key and returns the record.</p> <div class=highlight><pre><span></span><code>#!/usr/bin/env bash
|
||
|
|
|
||
|
|
showRecord() {
|
||
|
|
printf 'key[%d] = %d, %d\n' "$1" "${vals[@]:keys[$1]*2:2}"
|
||
|
|
}
|
||
|
|
|
||
|
|
parseRecords() {
|
||
|
|
trap 'unset -f _f' RETURN
|
||
|
|
_f() {
|
||
|
|
local x
|
||
|
|
IFS=: read -r _ x
|
||
|
|
((keys[x]=n++))
|
||
|
|
}
|
||
|
|
local n
|
||
|
|
|
||
|
|
_f
|
||
|
|
mapfile -tc2 -C _f "$1"
|
||
|
|
eval "$1"'=("${'"$1"'[@]##*:}")' # Return the array with some modification
|
||
|
|
}
|
||
|
|
|
||
|
|
main() {
|
||
|
|
local -a keys vals
|
||
|
|
parseRecords vals
|
||
|
|
showRecord "$1"
|
||
|
|
}
|
||
|
|
|
||
|
|
main "$1" <<-"EOF"
|
||
|
|
fabric.domain:123
|
||
|
|
routex:1
|
||
|
|
routey:2
|
||
|
|
fabric.domain:321
|
||
|
|
routex:6
|
||
|
|
routey:4
|
||
|
|
EOF
|
||
|
|
</code></pre></div> <p>For example, running <code>scriptname 321</code> would output <code>key[321] = 6, 4</code>. Every 2 lines read by <code>mapfile</code>, the function <code>_f</code> is called, which reads one additional line. Since the first line in the file is a key, and <code>_f</code> is responsible for the keys, it gets called first so that <code>mapfile</code> starts by reading the second line of input, calling <code>_f</code> with each subsequent 2 iterations. The RETURN trap is unimportant.</p> <h2 id=bugs>Bugs<a class=headerlink href=#bugs title="Permanent link">¶</a></h2> <ul> <li>Early implementations were buggy. For example, <code>mapfile</code> filling the readline history buffer with calls to the <code>CALLBACK</code>. This was fixed in 4.1 beta.</li> <li><code>mapfile -n</code> reads an extra line beyond the last line assigned to the array, through Bash. <a href=ftp://ftp.gnu.org/gnu/bash/bash-4.2-patches/bash42-035>Fixed in 4.2.35</a>.</li> <li><code>mapfile</code> callbacks could cause a crash if the variable being assigned is manipulated in certain ways. <a href=https://lists.gnu.org/archive/html/bug-bash/2013-01/msg00039.html>https://lists.gnu.org/archive/html/bug-bash/2013-01/msg00039.html</a>. Fixed in 4.3.</li> </ul> <h2 id=to-do>To Do<a class=headerlink href=#to-do title="Permanent link">¶</a></h2> <ul> <li>Create an implementation as a shell function that's portable between Ksh, Zsh, and Bash (and possibly other bourne-like shells with array support).</li> </ul> <h2 id=see-also>See also<a class=headerlink href=#see-also title="Permanent link">¶</a></h2> <ul> <li><a href=../../../syntax/arrays/ >arrays</a></li> <li><a href=../read/ >read</a> - If you don't know about this yet, why are you reading this page?</li> <li><a href=http://mywiki.wooledge.org/BashFAQ/001>http://mywiki.wooledge.org/BashFAQ/001</a> - It's FAQ 1 for a reason.</li> </ul> <aside class=md-source-file> <span class=md-source-file__fact> <span class=md-icon title="Last update"> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg> </span> <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 12, 2024</span> </span> <span class=md-source-file__fact> <span class=md-icon title=Created> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg> </span> <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 12, 2024</span> </span> </aside> <h2 id=__comments>Comments</h2> <script src=https://giscus.app/client.js data-repo=flokoe/bash-hackers-wiki data-repo-id=R_kgDOJ3Nr6Q data-category="Giscus Page Comments" data-category-id=DIC_kwDOJ3Nr6c4CXq9t data-mapping=pathname data-strict=1 data-reactions-enabled=1 data-emit-metadata=0 data-input-position=top data-theme=preferred_color_scheme data-lang=en data-loading=lazy crossorigin=anonymous async>
|
||
|
|
</script> <script>
|
||
|
|
var giscus = document.querySelector("script[src*=giscus]")
|
||
|
|
|
||
|
|
/* Set palette on initial load */
|
||
|
|
var palette = __md_get("__palette")
|
||
|
|
if (palette && typeof palette.color === "object") {
|
||
|
|
var theme = palette.color.scheme === "slate" ? "dark" : "light"
|
||
|
|
giscus.setAttribute("data-theme", theme)
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Register event handlers after documented loaded */
|
||
|
|
document.addEventListener("DOMContentLoaded", function() {
|
||
|
|
var ref = document.querySelector("[data-md-component=palette]")
|
||
|
|
ref.addEventListener("change", function() {
|
||
|
|
var palette = __md_get("__palette")
|
||
|
|
if (palette && typeof palette.color === "object") {
|
||
|
|
var theme = palette.color.scheme === "slate" ? "dark" : "light"
|
||
|
|
|
||
|
|
/* Instruct Giscus to change theme */
|
||
|
|
var frame = document.querySelector(".giscus-frame")
|
||
|
|
frame.contentWindow.postMessage(
|
||
|
|
{ giscus: { setConfig: { theme } } },
|
||
|
|
"https://giscus.app"
|
||
|
|
)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
})
|
||
|
|
</script> </article> </div> <script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script> </div> <button type=button class="md-top md-icon" data-md-component=top hidden> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg> Back to top </button> </main> <footer class=md-footer> <div class="md-footer-meta md-typeset"> <div class="md-footer-meta__inner md-grid"> <div class=md-copyright> Made with <a href=https://squidfunk.github.io/mkdocs-material/ target=_blank rel=noopener> Material for MkDocs </a> </div> </div> </div> </footer> </div> <div class=md-dialog data-md-component=dialog> <div class="md-dialog__inner md-typeset"></div> </div> <script id=__config type=application/json>{"base": "../../..", "features": ["navigation.instant", "navigation.tracking", "navigation.tabs", "navigation.sections", "navigation.top", "content.action.view", "content.action.edit", "search.suggest", "search.highlight", "content.code.copy"], "search": "../../../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script> <script src=../../../assets/javascripts/bundle.525ec568.min.js></script> </body> </html>
|