Remove all duplicate records from the input.

Declare unique flag (unique)

Variable declaration

Notes

This declaration relies on the C Virtual Machine to initialize the variable unique to zero.

Segment Source

 195: /* Flag to remove consecutive duplicate lines from the output.
 196:    Only the last of a sequence of equal lines will be output. */
 197: static int unique;
 198: 

Insert user message text

Text insertion

Segment Source

 236:   -u               with -c, check for strict ordering;\n\
 237:                    with -m, only output the first of an equal sequence\n\

Test field content only

Code modification

Avoid the last ditch byte-wise record comparision. With unique (and keys) enabled, comparision results depend solely on the results of the key comparision.

Segment Source

1180:       if (unique || stable)
1181:         return 0;

Check mode unique support

The semantics of the unique flag in the check only mode specify that no consecutive records should compare as equal.

Notes

This changes the semantics of the out-of-order test, but does not change the out-of-order action.

If duplicate records occur in separate files, CheckMode will fail to detect this.

Segment Element

Code modification

1241:           if ((unique && cmp >= 0) || (cmp > 0))

Segment Element

Code modification

1273:       if ((unique && cmp >= 0) || (cmp > 0))

Merge mode unique support

Eliminate duplicate records in merge mode.

Notes

Although this is scattered through mergefps() routine, it's function is very similar to the large file support in checkfp().

Segment Element

Structure declaration

1296:   struct line saved;            /* Saved line for unique check. */

Segment Element

Variable definition

1297:   int savedflag = 0;            /* True if there is a saved line. */

Segment Element

Variable declaration

1298:   int savealloc;                /* Size allocated for the saved line. */

Segment Element

Code insertion

1304:   register int i, j, t;
1305: 
1306: #ifdef lint  /* Suppress `used before initialized' warning.  */
1307:   savealloc = 0;
1308: #endif
1309: 
1310:   /* Allocate space for a saved line if necessary. */
1311:   if (unique)
1312:     {
1313:       savealloc = linelength;
1314:       saved.text = xmalloc (savealloc);
1315:     }
1316: 

Segment Element

Code insertion

1355:       if (unique)
1356:         {
1357:           if (savedflag && compare (&saved, &lines[ord[0]].lines[cur[ord[0]]]))
1358:             {
1359:               write_bytes (saved.text, saved.length, ofp);
1360:               putc (eolchar, ofp);
1361:               savedflag = 0;
1362:             }
1363:           if (!savedflag)
1364:             {
1365:               if (savealloc < lines[ord[0]].lines[cur[ord[0]]].length + 1)
1366:                 {
1367:                   while (savealloc < lines[ord[0]].lines[cur[ord[0]]].length + 1)
1368:                     savealloc *= 2;
1369:                   saved.text = xrealloc (saved.text, savealloc);
1370:                 }
1371:               saved.length = lines[ord[0]].lines[cur[ord[0]]].length;
1372:               memcpy (saved.text, lines[ord[0]].lines[cur[ord[0]]].text,
1373:                      saved.length + 1);
1374:               if (lines[ord[0]].lines[cur[ord[0]]].keybeg != NULL)
1375:                 {
1376:                   saved.keybeg = saved.text +
1377:                     (lines[ord[0]].lines[cur[ord[0]]].keybeg
1378:                      - lines[ord[0]].lines[cur[ord[0]]].text);
1379:                 }
1380:               if (lines[ord[0]].lines[cur[ord[0]]].keylim != NULL)
1381:                 {
1382:                   saved.keylim = saved.text +
1383:                     (lines[ord[0]].lines[cur[ord[0]]].keylim
1384:                      - lines[ord[0]].lines[cur[ord[0]]].text);
1385:                 }
1386:               savedflag = 1;
1387:             }
1388:         }
1389:       else
1390:         {

Segment Element

Code insertion

1394:         }

Segment Element

Code insertion

1443:   if (unique && savedflag)
1444:     {
1445:       write_bytes (saved.text, saved.length, ofp);
1446:       putc (eolchar, ofp);
1447:       free (saved.text);
1448:     }

Sort mode unique support

Code insertion

Eliminate duplicate records during a sort.

Notes

Because sortlines() does not eliminate duplicates, UniqueOpt support in sort() adds an extra test for every record.

Segment Source

1595:             if (!unique || i == 0
1596:                 || compare (&lines.lines[i], &lines.lines[i - 1]))

Parse UniqueOpt option (-u)

Code insertion

Segment Source

2000:                   case 'u':
2001:                     unique = 1;
2002:                     break;