/* sed.c - stream editor. Thing that does s/// and other stuff. * * Copyright 2014 Rob Landley * * See http://pubs.opengroup.org/onlinepubs/9699919799/utilities/sed.html USE_SED(NEWTOY(sed, "(version)e*f*inr", TOYFLAG_USR|TOYFLAG_BIN|TOYFLAG_LOCALE)) config SED bool "sed" default n help usage: sed [-inr] [-e SCRIPT]...|SCRIPT [-f SCRIPT_FILE]... [FILE...] Stream editor. Apply one or more editing SCRIPTs to each line of input (from FILE or stdin) producing output (by default to stdout). -e add SCRIPT to list -f add contents of SCRIPT_FILE to list -i Edit each file in place. -n No default output. (Use the p command to output matched lines.) -r Use extended regular expression syntax. -s Treat input files separately (implied by -i) A SCRIPT is a series of one or more COMMANDs separated by newlines or semicolons. All -e SCRIPTs are concatenated together as if separated by newlines, followed by all lines from -f SCRIPT_FILEs, in order. If no -e or -f SCRIPTs are specified, the first argument is the SCRIPT. Each COMMAND may be preceded by an address which limits the command to apply only to the specified line(s). Commands without an address apply to every line. Addresses are of the form: [ADDRESS[,ADDRESS]]COMMAND The ADDRESS may be a decimal line number (starting at 1), a /regular expression/ within a pair of forward slashes, or the character "$" which matches the last line of input. (In -s or -i mode this matches the last line of each file, otherwise just the last line of the last file.) A single address matches one line, a pair of comma separated addresses match everything from the first address to the second address (inclusive). If both addresses are regular expressions, more than one range of lines in each file can match. REGULAR EXPRESSIONS in sed are started and ended by the same character (traditionally / but anything except a backslash or a newline works). Backslashes may be used to escape the delimiter if it occurs in the regex, and for the usual printf escapes (\abcefnrtv and octal, hex, and unicode). An empty regex repeats the previous one. ADDRESS regexes (above) require the first delimeter to be escaped with a backslash when it isn't a forward slash (to distinguish it from the COMMANDs below). Sed mostly operates on individual lines one at a time. It reads each line, processes it, and either writes it to the output or discards it before reading the next line. Sed can remember one additional line in a separate buffer (using the h, H, g, G, and x commands), and can read the next line of input early (using the n and N command), but other than that command scripts operate on individual lines of text. Each COMMAND starts with a single character. The following commands take no arguments: { Start a new command block, continuing until a corresponding "}". Command blocks may nest. If the block has an address, commands within the block are only run for lines within the block's address range. } End command block (this command cannot have an address) d Delete this line and move on to the next one (ignores remaining COMMANDs) D Delete one line of input and restart command SCRIPT (same as "d" unless you've glued lines together with "N" or similar) g Get remembered line (overwriting current line) G Get remembered line (appending to current line) h Remember this line (overwriting remembered line) H Remember this line (appending to remembered line, if any) l Print this line, escaping \abfrtv (but leaving \n as a newline), using octal escapes for other nonprintable characters, and wrapping lines to terminal width with a backslash and newline n Print default output and read next line, replacing current line (If no next line available, quit processing script) N Append next line of input to this line, separated by a newline (This advances the line counter for address matching and "=", if no next line available quit processing script without default output) p Print this line P Print this line up to first newline (from "N") q Quit (print default output, no more commands processed or lines read) x Exchange this line with remembered line (overwrite in both directions) = Print the current line number (followed by a newline) The following commands (may) take an argument. The "text" arguments (to the "a", "b", and "c" commands) may end with an unescaped "\" to append the next line (for which leading whitespace is not skipped), and also treat ";" as a literal character (use "\;" instead). a [text] Append text to output before attempting to read next line b [label] Branch, jumps to :label (or with no label, to end of SCRIPT) c [text] Delete line, output text at end of matching address range (ignores remaining COMMANDs) i [text] Print text r [file] Append contents of file to output before attempting to read next line. s/S/R/F Search for regex S, replace matched text with R using flags F. The first character after the "s" (anything but newline or backslash) is the delimiter, escape with \ to use normally. The replacement text may contain "&" to substitute the matched text (escape it with backslash for a literal &), or \1 through \9 to substitute a parenthetical subexpression in the regex. You can also use the normal backslash escapes such as \n and a backslash at the end of the line appends the next line. The flags are: [0-9] A number, substitute only that occurrence of pattern g Global, substitute all occurrences of pattern i Ignore case when matching p Print the line if match was found and replaced w [file] Write (append) line to file if match replaced t [label] Test, jump to :label only if an "s" command found a match in this line since last test (replacing with same text counts) T [label] Test false, jump only if "s" hasn't found a match. w [file] Write (append) line to file y/old/new/ Change each character in 'old' to corresponding character in 'new' (with standard backslash escapes, delimiter can be any repeated character except \ or \n) : [label] Labeled target for jump commands # Comment, ignore rest of this line of SCRIPT Deviations from posix: allow extended regular expressions with -r, editing in place with -i, separate with -s, printf escapes in text, line continuations, semicolons after all commands, 2-address anywhere an address is allowed, "T" command, multiline continuations for [abc], \; to end [abc] argument before end of line. */ #define FOR_sed #include "toys.h" GLOBALS( struct arg_list *f; struct arg_list *e; // processed pattern list struct double_list *pattern; char *nextline, *remember; void *restart; long nextlen, rememberlen, count; int fdout, noeol; ) struct step { struct step *next, *prev; // Begin and end of each match long lmatch[2]; int rmatch[2], arg1, arg2, w; // offsets because remalloc() unsigned not, hit, sflags; char c; // action }; // Write out line with potential embedded NUL, handling eol/noeol static int emit(char *line, long len, int eol) { if (TT.noeol && !writeall(TT.fdout, "\n", 1)) return 1; if (eol) line[len++] = '\n'; TT.noeol = len && !eol; if (len && len != writeall(TT.fdout, line, len)) { perror_msg("short write"); return 1; } return 0; } // Do regex matching handling embedded NUL bytes in string. Note that // neither the pattern nor the match can currently include NUL bytes // (even with wildcards) and string must be nul terminated. static int ghostwheel(regex_t *preg, char *string, long len, int nmatch, regmatch_t pmatch[], int eflags) { /* while (len && !*string) { string++; len--; int l = strlen(string); if (len != strlen(string)) // todo: this long start = 0, rc = 0, matches = 0; for (;;) { long new = strlen(string+start); // eflags nobegin noend rc |= regexec(preg, string+start, nmatch-matches, pmatch+matches, eflags); if ((start += end + 1) >= len) break; } return rc; */ return regexec(preg, string, nmatch, pmatch, eflags); } // Extend allocation to include new string, with newline between if newlen<0 static char *extend_string(char **old, char *new, int oldlen, int newlen) { int newline = newlen < 0; char *s; if (newline) newlen = -newlen; s = *old = xrealloc(*old, oldlen+newlen+newline+1); if (newline) s[oldlen++] = '\n'; memcpy(s+oldlen, new, newlen); s[oldlen+newlen] = 0; return s+oldlen+newlen; } // Apply pattern to line from input file static void walk_pattern(char **pline, long plen) { struct append { struct append *next, *prev; int file; char *str; } *append = 0; char *line = TT.nextline; long len = TT.nextlen; struct step *logrus; int eol = 0, tea = 0; // Grab next line for deferred processing (EOF detection: we get a NULL // pline at EOF to flush last line). Note that only end of _last_ input // file matches $ (unless we're doing -i). if (pline) { TT.nextline = *pline; TT.nextlen = plen; *pline = 0; } if (!line || !len) return; if (line[len-1] == '\n') line[--len] = eol++; TT.count++; logrus = TT.restart ? TT.restart : (void *)TT.pattern; TT.restart = 0; while (logrus) { char *str, c = logrus->c; // Have we got a matching range for this rule? if (*logrus->lmatch || *logrus->rmatch) { int miss = 0; long lm; // In a match that might end? if (logrus->hit) { if (!(lm = logrus->lmatch[1])) { if (!logrus->rmatch[1]) logrus->hit = 0; else { void *rm = logrus->rmatch[1] + (char *)logrus; // regex match end includes matching line, so defer deactivation if (!ghostwheel(rm, line, len, 0, 0, 0)) miss = 1; } } else if (lm > 0 && lm < TT.count) logrus->hit = 0; // Start a new match? } else { if (!(lm = *logrus->lmatch)) { void *rm = *logrus->rmatch + (char *)logrus; if (!ghostwheel(rm, line, len, 0, 0, 0)) logrus->hit++; } else if (lm == TT.count || (lm == -1 && !pline)) logrus->hit++; } // Didn't match? if (!(logrus->hit ^ logrus->not)) { // Handle skipping curly bracket command group if (c == '{') { int curly = 1; while (curly) { logrus = logrus->next; if (logrus->c == '{') curly++; if (logrus->c == '}') curly--; } } logrus = logrus->next; continue; } // Deferred disable from regex end match if (miss) logrus->hit = 0; } // Process command if (c=='a' || c=='r') { struct append *a = xzalloc(sizeof(struct append)); a->str = logrus->arg1+(char *)logrus; a->file = c== 'r'; dlist_add_nomalloc((void *)&append, (void *)a); } else if (c=='b' || c=='t' || c=='T') { int t = tea; if (c != 'b') tea = 0; if (c=='b' || t^(c=='T')) { str = logrus->arg1+(char *)logrus; if (!*str) break; for (logrus = (void *)TT.pattern; logrus; logrus = logrus->next) if (logrus->c == ':' && !strcmp(logrus->arg1+(char *)logrus, str)) break; if (!logrus) error_exit("no :%s", str); } } else if (c=='c') { str = logrus->arg1+(char *)logrus; if (!logrus->hit) emit(str, strlen(str), 1); goto done; } else if (c=='d') goto done; else if (c=='D') { // Delete up to \n or end of buffer for (str = line; !*str || *str=='\n'; str++); len -= str - line; memmove(line, str, len); line[len] = 0; // restart script logrus = (void *)TT.pattern; continue; } else if (c=='g') { free(line); line = xstrdup(TT.remember); len = TT.rememberlen; } else if (c=='G') { line = xrealloc(line, len+TT.rememberlen+2); line[len++] = '\n'; memcpy(line+len, TT.remember, TT.rememberlen); line[len += TT.rememberlen] = 0; } else if (c=='h') { free(TT.remember); TT.remember = xstrdup(line); TT.rememberlen = len; } else if (c=='H') { TT.remember = xrealloc(TT.remember, TT.rememberlen+len+2); TT.remember[TT.rememberlen++] = '\n'; memcpy(TT.remember+TT.rememberlen, line, len); TT.remember[TT.rememberlen += len] = 0; } else if (c=='i') { str = logrus->arg1+(char *)logrus; emit(str, strlen(str), 1); // } else if (c=='l') { // error_exit("todo: l"); } else if (c=='n') { TT.restart = logrus->next; break; } else if (c=='N') { if (pline) { TT.restart = logrus->next; extend_string(&line, TT.nextline, plen, -TT.nextlen); free(TT.nextline); TT.nextline = line; line = 0; } goto done; } else if (c=='p') { if (emit(line, len, eol)) break; } else if (c=='q') break; else if (c=='s') { char *rline = line, *new = logrus->arg2 + (char *)logrus;; regmatch_t *match = (void *)toybuf; regex_t *reg = (void *)(logrus->arg1+(char *)logrus); unsigned mflags = 0, i = 0, rlen = len, mlen, off, newlen; // Find match while (!ghostwheel(reg, rline, rlen, 10, match, mflags)) { mflags = REG_NOTBOL; // Zero length matches don't count immediately after last match mlen = match[0].rm_eo-match[0].rm_so; if (!mlen && !match[0].rm_so) { if (!rlen--) break; rline++; continue; } // Replace only a specific match? off = logrus->sflags>>3; if (off && off != ++i) { rline += match[0].rm_eo; rlen -= match[0].rm_eo; continue; } // The strlen() and memcpy don't support \1 or & yet. newlen = strlen(new); len = len-mlen+newlen; off = rline-line; line = xrealloc(line, len+1); rline = line+off; rlen -= match[0].rm_eo; memmove(rline+match[0].rm_so+newlen, rline+match[0].rm_eo, rlen+1); memcpy(rline+match[0].rm_so, new, newlen); rline += newlen+match[0].rm_so; // flag g if (!(logrus->sflags & 2)) break; } if (mflags) { // flag p if (logrus->sflags & 4) emit(line, len, eol); tea = 1; if (logrus->w) goto writenow; } } else if (c=='w') { int fd, noeol; char *name; writenow: // Swap out emit() context fd = TT.fdout; noeol = TT.noeol; // We save filehandle and newline status before filename name = logrus->w + (char *)logrus; memcpy(&TT.fdout, name, 4); name += 4; TT.noeol = *(name++); // write, then save/restore context if (emit(line, len, eol)) perror_exit("w '%s'", logrus->arg1+(char *)logrus); *(--name) = TT.noeol; TT.noeol = noeol; TT.fdout = fd; } else if (c=='x') { long swap = TT.rememberlen; str = TT.remember; TT.remember = line; line = str; TT.rememberlen = len; len = swap; } else if (c=='y') { char *from, *to = (char *)logrus; int i, j; from = to+logrus->arg1; to += logrus->arg2; for (i = 0; i < len; i++) { j = stridx(from, line[i]); if (j != -1) line[i] = to[j]; } } else if (c=='=') xprintf("%ld\n", TT.count); else if (c!=':') error_exit("todo: %c", c); logrus = logrus->next; } if (!(toys.optflags & FLAG_n)) emit(line, len, eol); done: free(line); if (dlist_terminate(append)) while (append) { struct append *a = append->next; if (append->file) { int fd = xopen(append->str, O_RDONLY); // Force newline if noeol pending emit(0, 0, 0); xsendfile(fd, TT.fdout); close(fd); } else emit(append->str, strlen(append->str), 1); free(append); append = a; } } // Genericish function, can probably get moved to lib.c // Iterate over lines in file, calling function. Function can write NULL to // the line pointer if they want to keep it, otherwise line is freed. // Passed file descriptor is closed at the end of processing. static void do_lines(int fd, char *name, void (*call)(char **pline, long len)) { FILE *fp = fd ? xfdopen(fd, "r") : stdin; for (;;) { char *line = 0; ssize_t len; len = getline(&line, (void *)&len, fp); if (len > 0) { call(&line, len); free(line); } else break; } if (fd) fclose(fp); } // Iterate over newline delimited data blob (potentially with embedded NUL), // call function on each line. static void chop_lines(char *data, long len, void (*call)(char **p, long l)) { long ll; for (ll = 0; ll < len; ll++) { if (data[ll] == '\n') { char *c = data; data[ll] = 0; call(&c, len); data[ll++] = '\n'; data += ll; len -= ll; ll = -1; } } if (len) call(&data, len); } static void do_sed(int fd, char *name) { int i = toys.optflags & FLAG_i; if (i) { // todo: rename dance } do_lines(fd, name, walk_pattern); if (i) { walk_pattern(0, 0); // todo: rename dance } } // Note: removing backslash escapes and null terminating edits the source // string, which could be from the environment space via -e, which could // screw up what "ps" sees, and I'm ok with that. (Modifying the environment // space like that means sed is very, very not reentrant.) // Ok, what happens if we xexec() sed with constant arguments then? // TODO: ^^^ that // returns length of processed string, *pstr advances to next unused char, // if delim (or *delim) is 0 uses starting char as delimiter, otherwise // parses and saves delimiter from first character(s) static int unescape_delimited_string(char **pstr, char *delim) { char *to, *from = *pstr, d; int rc; if (!delim || !*delim) { if (!(d = *(from++))) return -1; if (d == '\\') d = *(from++); if (!d || d == '\\') return -1; if (delim) *delim = d; } else d = *delim; for (to = *pstr; *from != d; *(to++) = *(from++)) { if (!*from) return -1; if (*from == '\\') { if (!from[1]) return -1; // Check escaped end delimiter before printf style escapes. if (from[1] == d) from++; else { char c = unescape(from[1]); if (c) { *to = c; from++; } } } } rc = to-*pstr; *to = 0; *pstr = from+1; return rc; } // Translate primal pattern into walkable form. static void jewel_of_judgement(char **pline, long len) { struct step *corwin = (void *)TT.pattern; char *line = *pline, *reg, c; int i; // Append additional line to pattern argument string? if (corwin && corwin->prev->hit) { // Remove half-finished entry from list so remalloc() doesn't confuse it TT.pattern = TT.pattern->prev; corwin = dlist_pop(&TT.pattern); corwin->hit = 0; c = corwin->c; reg = (char *)corwin; reg += corwin->arg1 + strlen(reg + corwin->arg1); // Resume parsing goto append; } // Loop through commands in line corwin = 0; for (;;) { if (corwin) dlist_add_nomalloc(&TT.pattern, (void *)corwin); while (isspace(*line) || *line == ';') line++; if (!*line || *line == '#') return; memset(toybuf, 0, sizeof(struct step)); corwin = (void *)toybuf; reg = toybuf + sizeof(struct step); // Parse address range (if any) for (i = 0; i < 2; i++) { if (*line == ',') line++; else if (i) break; if (isdigit(*line)) corwin->lmatch[i] = strtol(line, &line, 0); else if (*line == '$') { corwin->lmatch[i] = -1; line++; } else if (*line == '/' || *line == '\\') { char *s = line; if (-1 == unescape_delimited_string(&line, 0)) goto brand; xregcomp((void *)reg, s, ((toys.optflags & FLAG_r)*REG_EXTENDED)|REG_NOSUB); corwin->rmatch[i] = reg-toybuf; reg += sizeof(regex_t); } else break; } while (isspace(*line)) line++; if (!*line) break; while (*line == '!') { corwin->not = 1; line++; } while (isspace(*line)) line++; c = corwin->c = *(line++); if (strchr("}:", c) && i) break; if (strchr("aiqr=", c) && i>1) break; // Add step to pattern corwin = xmalloc(reg-toybuf); memcpy(corwin, toybuf, reg-toybuf); reg = (reg-toybuf) + (char *)corwin; // Parse arguments by command type if (c == '{') TT.nextlen++; else if (c == '}') { if (!TT.nextlen--) break; } else if (c == 's') { void *merlin, *fiona; int end; char delim = 0; // s/pattern/replacement/flags // get pattern (just record, we parse it later) corwin->arg1 = reg - (char *)corwin; merlin = line; if (-1 == unescape_delimited_string(&line, &delim)) goto brand; // get replacement fiona = line; if (-1 == (end = unescape_delimited_string(&line, &delim))) goto brand; corwin->arg2 = corwin->arg1 + sizeof(regex_t); reg = extend_string((void *)&corwin, fiona, corwin->arg2, end); // get flags for (; *line && *line != ';'; line++) { long l; if (isspace(*line)) continue; if (0 <= (l = stridx("igp", *line))) corwin->sflags |= 1<sflags >> 3 && 0<(l = strtol(line+end, &line, 10))) { corwin->sflags |= l << 3; line--; } else if (*line == 'w') break; else goto brand; } // We deferred actually parsing the rexex until we had the s///i flag // allocating the space was done by extend_string() above xregcomp((void *)(corwin->arg1 + (char *)corwin), merlin, ((toys.optflags & FLAG_r)*REG_EXTENDED)|((corwin->sflags&1)*REG_ICASE)); if (*line == 'w') { line++; goto writenow; } } else if (c == 'w') { int fd, delim; char *cc; // Since s/// uses arg1 and arg2, and w needs a persistent filehandle and // eol status, and to retain the filename for error messages, we'd need // to go up to arg5 just for this. Compromise: dynamically allocate the // filehandle and eol status. writenow: while (isspace(*line)) line++; if (!*line) goto brand; for (cc = line; *cc; cc++) if (*cc == '\\' && cc[1] == ';') break; delim = *cc; *cc = 0; fd = xcreate(line, O_WRONLY|O_CREAT|O_TRUNC, 0544); *cc = delim; delim = cc-line; corwin->w = reg - (char *)corwin; corwin = xrealloc(corwin, corwin->w + delim + 6); reg = corwin->w + (char *)corwin; memcpy(reg, &fd, 4); reg += 4; *(reg++) = 0; memcpy(reg, line, delim); reg += delim; *(reg++) = 0; line = cc; if (delim) line += 2; } else if (c == 'y') { char *s = line, delim = 0; int len1, len2; if (-1 == (len1 = unescape_delimited_string(&line, &delim))) goto brand; corwin->arg1 = reg-(char *)corwin; reg = extend_string((void *)&corwin, s, reg-(char *)corwin, len1); s = line; corwin->arg2 = reg-(char *)corwin; if (-1 == (len2 = unescape_delimited_string(&line, &delim))) goto brand; if (len1 != len2) goto brand; reg = extend_string((void *)&corwin, s, reg-(char*)corwin, len2); } else if (strchr("abcirtTw:", c)) { int end, class; // Trim whitespace from "b ;" and ": blah " but only first space in "w x " while (isspace(*line)) line++; append: class = !strchr("btT:", c); end = strcspn(line, class ? "" : "; \t\r\n\v\f"); if (!end) { if (!strchr("btT", c)) break; continue; } // Extend allocation to include new string. We use offsets instead of // pointers so realloc() moving stuff doesn't break things. Do it // here instead of toybuf so there's no maximum size. if (!corwin->arg1) corwin->arg1 = reg - (char*)corwin; reg = extend_string((void *)&corwin, line, reg - (char *)corwin, end); // Line continuation? if (class && reg[-1] == '\\') { reg[-1] = 0; corwin->hit++; } // Commands that take no arguments } else if (!strchr("{dDgGhHlnNpPqx=", *line)) break; } brand: // Reminisce about chestnut trees. error_exit("bad pattern '%s'@%ld (%c)", *pline, line-*pline+1, *line); } void sed_main(void) { struct arg_list *dworkin; char **args = toys.optargs; // Lie to autoconf when it asks stupid questions, so configure regexes // that look for "GNU sed version %f" greater than some old buggy number // don't fail us for not matching their narrow expectations. if (toys.optflags & FLAG_version) { xprintf("This is not GNU sed version 9.0\n"); return; } // Need a pattern. If no unicorns about, fight serpent and take its eye. if (!TT.e && !TT.f) { if (!*toys.optargs) error_exit("no pattern"); (TT.e = xzalloc(sizeof(struct arg_list)))->arg = *(args++); } // Option parsing infrastructure can't interlace "-e blah -f blah -e blah" // so handle all -e, then all -f. (At least the behavior's consistent.) for (dworkin = TT.e; dworkin; dworkin = dworkin->next) chop_lines(dworkin->arg, strlen(dworkin->arg), jewel_of_judgement); for (dworkin = TT.f; dworkin; dworkin = dworkin->next) do_lines(xopen(dworkin->arg, O_RDONLY), dworkin->arg, jewel_of_judgement); dlist_terminate(TT.pattern); if (TT.nextlen) error_exit("no }"); TT.fdout = 1; TT.remember = xstrdup(""); // Inflict pattern upon input files loopfiles_rw(args, O_RDONLY, 0, 0, do_sed); if (!(toys.optflags & FLAG_i)) walk_pattern(0, 0); // todo: need to close fd when done for TOYBOX_FREE? }