sfeed.c - sfeed - RSS and Atom parser
(HTM) git clone git://git.codemadness.org/sfeed
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
sfeed.c (30053B)
---
1 #include <errno.h>
2 #include <stdint.h>
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <string.h>
6 #include <strings.h>
7
8 #include "util.h"
9 #include "xml.h"
10
11 #define ISINCONTENT(ctx) ((ctx).iscontent && !((ctx).iscontenttag))
12 #define ISCONTENTTAG(ctx) (!((ctx).iscontent) && (ctx).iscontenttag)
13
14 /* these feed fields support multiple separated values */
15 #define ISFEEDFIELDMULTI(t) ((t) == FeedFieldCategory)
16
17 /* string and byte-length */
18 #define STRP(s) s,sizeof(s)-1
19
20 enum FeedType {
21 FeedTypeNone = 0,
22 FeedTypeRSS = 1,
23 FeedTypeAtom = 2
24 };
25
26 enum ContentType {
27 ContentTypeNone = 0,
28 ContentTypePlain = 1,
29 ContentTypeHTML = 2
30 };
31 static const char *contenttypes[] = { "", "plain", "html" };
32
33 /* String data / memory pool */
34 typedef struct string {
35 char *data; /* data */
36 size_t len; /* string length */
37 size_t bufsiz; /* allocated size */
38 } String;
39
40 /* NOTE: the order of these fields (content, date, author) indicate the
41 * priority to use them, from least important to high. */
42 enum TagId {
43 TagUnknown = 0,
44 /* RSS */
45 RSSTagDcdate, RSSTagPubdate, /* creation date has higher priority */
46 RSSTagTitle,
47 RSSTagMediaDescription, RSSTagDescription, RSSTagContentEncoded,
48 RSSTagGuid,
49 RSSTagGuidPermalinkFalse,
50 RSSTagGuidPermalinkTrue,
51 /* must be defined after GUID, because it can be a link (isPermaLink) */
52 RSSTagLink,
53 RSSTagEnclosure,
54 RSSTagAuthor, RSSTagDccreator,
55 RSSTagCategory,
56 /* Atom */
57 /* creation date has higher priority */
58 AtomTagModified, AtomTagUpdated, AtomTagIssued, AtomTagPublished,
59 AtomTagTitle,
60 AtomTagMediaDescription, AtomTagSummary, AtomTagContent,
61 AtomTagId,
62 AtomTagLink,
63 AtomTagLinkAlternate,
64 AtomTagLinkEnclosure,
65 AtomTagAuthor, AtomTagAuthorName,
66 AtomTagCategory,
67 TagLast
68 };
69
70 typedef struct feedtag {
71 char *name; /* name of tag to match */
72 size_t len; /* len of `name` */
73 enum TagId id; /* unique ID */
74 } FeedTag;
75
76 typedef struct field {
77 String str;
78 enum TagId tagid; /* tagid set previously, used for tag priority */
79 } FeedField;
80
81 enum {
82 FeedFieldTime = 0, FeedFieldTitle, FeedFieldLink, FeedFieldContent,
83 FeedFieldId, FeedFieldAuthor, FeedFieldEnclosure, FeedFieldCategory,
84 FeedFieldLast
85 };
86
87 typedef struct feedcontext {
88 String *field; /* current FeedItem field String */
89 FeedField fields[FeedFieldLast]; /* data for current item */
90 FeedTag tag; /* unique current parsed tag */
91 int iscontent; /* in content data */
92 int iscontenttag; /* in content tag */
93 enum ContentType contenttype; /* content-type for item */
94 enum FeedType feedtype;
95 int attrcount; /* count item HTML element attributes */
96 } FeedContext;
97
98 static long long datetounix(long long, int, int, int, int, int);
99 static FeedTag * gettag(enum FeedType, const char *, size_t);
100 static long gettzoffset(const char *);
101 static int isattr(const char *, size_t, const char *, size_t);
102 static int istag(const char *, size_t, const char *, size_t);
103 static int parsetime(const char *, long long *);
104 static void printfields(void);
105 static void string_append(String *, const char *, size_t);
106 static void string_buffer_realloc(String *, size_t);
107 static void string_clear(String *);
108 static void string_print_encoded(String *);
109 static void string_print_timestamp(String *);
110 static void string_print_trimmed(String *);
111 static void string_print_trimmed_multi(String *);
112 static void string_print_uri(String *);
113 static void xmlattr(XMLParser *, const char *, size_t, const char *, size_t,
114 const char *, size_t);
115 static void xmlattrentity(XMLParser *, const char *, size_t, const char *,
116 size_t, const char *, size_t);
117 static void xmlattrend(XMLParser *, const char *, size_t, const char *,
118 size_t);
119 static void xmlattrstart(XMLParser *, const char *, size_t, const char *,
120 size_t);
121 static void xmldata(XMLParser *, const char *, size_t);
122 static void xmldataentity(XMLParser *, const char *, size_t);
123 static void xmltagend(XMLParser *, const char *, size_t, int);
124 static void xmltagstart(XMLParser *, const char *, size_t);
125 static void xmltagstartparsed(XMLParser *, const char *, size_t, int);
126
127 /* map tag name to TagId type */
128 /* RSS, must be alphabetical order */
129 static const FeedTag rsstags[] = {
130 { STRP("author"), RSSTagAuthor },
131 { STRP("category"), RSSTagCategory },
132 { STRP("content:encoded"), RSSTagContentEncoded },
133 { STRP("dc:creator"), RSSTagDccreator },
134 { STRP("dc:date"), RSSTagDcdate },
135 { STRP("description"), RSSTagDescription },
136 /* RSS: <enclosure url="" />, Atom has <link rel="enclosure" /> */
137 { STRP("enclosure"), RSSTagEnclosure },
138 { STRP("guid"), RSSTagGuid },
139 { STRP("link"), RSSTagLink },
140 { STRP("media:description"), RSSTagMediaDescription },
141 { STRP("pubdate"), RSSTagPubdate },
142 { STRP("title"), RSSTagTitle }
143 };
144
145 /* Atom, must be alphabetical order */
146 static const FeedTag atomtags[] = {
147 { STRP("author"), AtomTagAuthor },
148 { STRP("category"), AtomTagCategory },
149 { STRP("content"), AtomTagContent },
150 { STRP("id"), AtomTagId },
151 { STRP("issued"), AtomTagIssued }, /* Atom 0.3 */
152 /* Atom: <link href="" />, RSS has <link></link> */
153 { STRP("link"), AtomTagLink },
154 { STRP("media:description"), AtomTagMediaDescription },
155 { STRP("modified"), AtomTagModified }, /* Atom 0.3 */
156 { STRP("published"), AtomTagPublished },
157 { STRP("summary"), AtomTagSummary },
158 { STRP("title"), AtomTagTitle },
159 { STRP("updated"), AtomTagUpdated }
160 };
161
162 /* special case: nested <author><name> */
163 static const FeedTag atomtagauthor = { STRP("author"), AtomTagAuthor };
164 static const FeedTag atomtagauthorname = { STRP("name"), AtomTagAuthorName };
165
166 /* reference to no / unknown tag */
167 static const FeedTag notag = { STRP(""), TagUnknown };
168
169 /* map TagId type to RSS/Atom field, all tags must be defined */
170 static const int fieldmap[TagLast] = {
171 [TagUnknown] = -1,
172 /* RSS */
173 [RSSTagDcdate] = FeedFieldTime,
174 [RSSTagPubdate] = FeedFieldTime,
175 [RSSTagTitle] = FeedFieldTitle,
176 [RSSTagMediaDescription] = FeedFieldContent,
177 [RSSTagDescription] = FeedFieldContent,
178 [RSSTagContentEncoded] = FeedFieldContent,
179 [RSSTagGuid] = -1,
180 [RSSTagGuidPermalinkFalse] = FeedFieldId,
181 [RSSTagGuidPermalinkTrue] = FeedFieldId, /* special-case: both a link and an id */
182 [RSSTagLink] = FeedFieldLink,
183 [RSSTagEnclosure] = FeedFieldEnclosure,
184 [RSSTagAuthor] = FeedFieldAuthor,
185 [RSSTagDccreator] = FeedFieldAuthor,
186 [RSSTagCategory] = FeedFieldCategory,
187 /* Atom */
188 [AtomTagModified] = FeedFieldTime,
189 [AtomTagUpdated] = FeedFieldTime,
190 [AtomTagIssued] = FeedFieldTime,
191 [AtomTagPublished] = FeedFieldTime,
192 [AtomTagTitle] = FeedFieldTitle,
193 [AtomTagMediaDescription] = FeedFieldContent,
194 [AtomTagSummary] = FeedFieldContent,
195 [AtomTagContent] = FeedFieldContent,
196 [AtomTagId] = FeedFieldId,
197 [AtomTagLink] = -1,
198 [AtomTagLinkAlternate] = FeedFieldLink,
199 [AtomTagLinkEnclosure] = FeedFieldEnclosure,
200 [AtomTagAuthor] = -1,
201 [AtomTagAuthorName] = FeedFieldAuthor,
202 [AtomTagCategory] = FeedFieldCategory
203 };
204
205 static const int FieldSeparator = '\t';
206 /* separator for multiple values in a field, separator should be 1 byte */
207 static const char FieldMultiSeparator[] = "|";
208 static struct uri baseuri;
209 static const char *baseurl;
210
211 static FeedContext ctx;
212 static XMLParser parser; /* XML parser state */
213 static String attrispermalink, attrrel, attrtype, tmpstr;
214
215 static int
216 tagcmp(const void *v1, const void *v2)
217 {
218 return strcasecmp(((FeedTag *)v1)->name, ((FeedTag *)v2)->name);
219 }
220
221 /* Unique tagid for parsed tag name. */
222 static FeedTag *
223 gettag(enum FeedType feedtype, const char *name, size_t namelen)
224 {
225 FeedTag f, *r = NULL;
226
227 f.name = (char *)name;
228
229 switch (feedtype) {
230 case FeedTypeRSS:
231 r = bsearch(&f, rsstags, sizeof(rsstags) / sizeof(rsstags[0]),
232 sizeof(rsstags[0]), tagcmp);
233 break;
234 case FeedTypeAtom:
235 r = bsearch(&f, atomtags, sizeof(atomtags) / sizeof(atomtags[0]),
236 sizeof(atomtags[0]), tagcmp);
237 break;
238 default:
239 break;
240 }
241
242 return r;
243 }
244
245 static char *
246 ltrim(const char *s)
247 {
248 for (; ISSPACE((unsigned char)*s); s++)
249 ;
250 return (char *)s;
251 }
252
253 static char *
254 rtrim(const char *s)
255 {
256 const char *e;
257
258 for (e = s + strlen(s); e > s && ISSPACE((unsigned char)*(e - 1)); e--)
259 ;
260 return (char *)e;
261 }
262
263 /* Clear string only; don't free, prevents unnecessary reallocation. */
264 static void
265 string_clear(String *s)
266 {
267 if (s->data)
268 s->data[0] = '\0';
269 s->len = 0;
270 }
271
272 static void
273 string_buffer_realloc(String *s, size_t newlen)
274 {
275 size_t alloclen;
276
277 if (newlen > SIZE_MAX / 2) {
278 alloclen = SIZE_MAX;
279 } else {
280 for (alloclen = 64; alloclen <= newlen; alloclen *= 2)
281 ;
282 }
283 if (!(s->data = realloc(s->data, alloclen)))
284 err(1, "realloc");
285 s->bufsiz = alloclen;
286 }
287
288 /* Append data to String, s->data and data may not overlap. */
289 static void
290 string_append(String *s, const char *data, size_t len)
291 {
292 if (!len)
293 return;
294
295 if (s->len >= SIZE_MAX - len) {
296 errno = ENOMEM;
297 err(1, "realloc");
298 }
299
300 /* check if allocation is necessary, never shrink the buffer. */
301 if (s->len + len >= s->bufsiz)
302 string_buffer_realloc(s, s->len + len + 1);
303 memcpy(s->data + s->len, data, len);
304 s->len += len;
305 s->data[s->len] = '\0';
306 }
307
308 /* Print text, encode TABs, newlines and '\', remove other whitespace.
309 * Remove leading and trailing whitespace. */
310 static void
311 string_print_encoded(String *s)
312 {
313 const char *p, *e;
314
315 if (!s->data || !s->len)
316 return;
317
318 p = ltrim(s->data);
319 e = rtrim(p);
320
321 for (; *p && p != e; p++) {
322 switch (*p) {
323 case '\n': putchar('\\'); putchar('n'); break;
324 case '\\': putchar('\\'); putchar('\\'); break;
325 case '\t': putchar('\\'); putchar('t'); break;
326 default:
327 /* ignore control chars */
328 if (!ISCNTRL((unsigned char)*p))
329 putchar(*p);
330 break;
331 }
332 }
333 }
334
335 static void
336 printtrimmed(const char *s)
337 {
338 char *p, *e;
339
340 p = ltrim(s);
341 e = rtrim(p);
342 for (; *p && p != e; p++) {
343 if (ISSPACE((unsigned char)*p))
344 putchar(' '); /* any whitespace to space */
345 else if (!ISCNTRL((unsigned char)*p))
346 /* ignore other control chars */
347 putchar(*p);
348 }
349 }
350
351 /* Print text, replace TABs, carriage return and other whitespace with ' '.
352 * Other control chars are removed. Remove leading and trailing whitespace. */
353 static void
354 string_print_trimmed(String *s)
355 {
356 if (!s->data || !s->len)
357 return;
358
359 printtrimmed(s->data);
360 }
361
362 /* Print each field with trimmed whitespace, separated by '|'. */
363 static void
364 string_print_trimmed_multi(String *s)
365 {
366 char *p, *e;
367 int c;
368
369 if (!s->data || !s->len)
370 return;
371
372 for (p = s->data; ; p = e + 1) {
373 if ((e = strstr(p, FieldMultiSeparator))) {
374 c = *e;
375 *e = '\0';
376 printtrimmed(p);
377 *e = c; /* restore NUL byte to original character */
378 fputs(FieldMultiSeparator, stdout);
379 } else {
380 printtrimmed(p);
381 break;
382 }
383 }
384 }
385
386 /* Print URL, if it is a relative URL then it uses the global `baseurl`. */
387 static void
388 printuri(char *s)
389 {
390 char link[4096], *p, *e;
391 struct uri newuri, olduri;
392 int c, r = -1;
393
394 p = ltrim(s);
395 e = rtrim(p);
396 c = *e;
397 *e = '\0';
398
399 if (baseurl && !uri_hasscheme(p) &&
400 uri_parse(p, &olduri) != -1 && !olduri.proto[0] &&
401 uri_makeabs(&newuri, &olduri, &baseuri) != -1 && newuri.proto[0])
402 r = uri_format(link, sizeof(link), &newuri);
403
404 if (r >= 0 && (size_t)r < sizeof(link))
405 printtrimmed(link);
406 else
407 printtrimmed(p);
408
409 *e = c; /* restore NUL byte to original character */
410 }
411
412 /* Print URL, if it is a relative URL then it uses the global `baseurl`. */
413 static void
414 string_print_uri(String *s)
415 {
416 if (!s->data || !s->len)
417 return;
418
419 printuri(s->data);
420 }
421
422 /* Print as UNIX timestamp, print nothing if the time is empty or invalid. */
423 static void
424 string_print_timestamp(String *s)
425 {
426 long long t;
427
428 if (!s->data || !s->len)
429 return;
430
431 if (parsetime(s->data, &t) != -1)
432 printf("%lld", t);
433 }
434
435 /* Convert time fields. Returns a signed (at least) 64-bit UNIX timestamp.
436 Parameters should be passed as they are in a struct tm:
437 that is: year = year - 1900, month = month - 1. */
438 static long long
439 datetounix(long long year, int mon, int day, int hour, int min, int sec)
440 {
441 /* seconds in a month in a regular (non-leap) year */
442 static const long secs_through_month[] = {
443 0, 31 * 86400, 59 * 86400, 90 * 86400,
444 120 * 86400, 151 * 86400, 181 * 86400, 212 * 86400,
445 243 * 86400, 273 * 86400, 304 * 86400, 334 * 86400 };
446 int is_leap = 0, cycles, centuries = 0, leaps = 0, rem;
447 long long t;
448
449 /* optimization: handle common range year 1902 up to and including 2038 */
450 if (year - 2ULL <= 136) {
451 /* amount of leap days relative to 1970: every 4 years */
452 leaps = (year - 68) >> 2;
453 if (!((year - 68) & 3)) {
454 leaps--;
455 is_leap = 1;
456 } else {
457 is_leap = 0;
458 }
459 t = 31536000 * (year - 70) + (86400 * leaps); /* 365 * 86400 = 31536000 */
460 } else {
461 /* general leap year calculation:
462 leap years occur mostly every 4 years but every 100 years
463 a leap year is skipped unless the year is divisible by 400 */
464 cycles = (year - 100) / 400;
465 rem = (year - 100) % 400;
466 if (rem < 0) {
467 cycles--;
468 rem += 400;
469 }
470 if (!rem) {
471 is_leap = 1;
472 } else {
473 if (rem >= 300) {
474 centuries = 3;
475 rem -= 300;
476 } else if (rem >= 200) {
477 centuries = 2;
478 rem -= 200;
479 } else if (rem >= 100) {
480 centuries = 1;
481 rem -= 100;
482 }
483 if (rem) {
484 leaps = rem / 4U;
485 rem %= 4U;
486 is_leap = !rem;
487 }
488 }
489 leaps += (97 * cycles) + (24 * centuries) - is_leap;
490
491 /* adjust 8 leap days from 1970 up to and including 2000:
492 ((30 * 365) + 8) * 86400 = 946771200 */
493 t = ((year - 100) * 31536000LL) + (leaps * 86400LL) + 946771200LL;
494 }
495 t += secs_through_month[mon];
496 if (is_leap && mon >= 2)
497 t += 86400;
498 t += 86400LL * (day - 1);
499 t += 3600LL * hour;
500 t += 60LL * min;
501 t += sec;
502
503 return t;
504 }
505
506 /* Get timezone from string, return time offset in seconds from UTC.
507 * NOTE: only parses timezones in RFC 822, many other timezone names are
508 * ambiguous anyway.
509 * ANSI and military zones are defined wrong in RFC 822 and are unsupported,
510 * see note on RFC 2822 4.3 page 32. */
511 static long
512 gettzoffset(const char *s)
513 {
514 static const struct {
515 char *name;
516 int offhour;
517 } tzones[] = {
518 { "CDT", -5 * 3600 },
519 { "CST", -6 * 3600 },
520 { "EDT", -4 * 3600 },
521 { "EST", -5 * 3600 },
522 { "MDT", -6 * 3600 },
523 { "MST", -7 * 3600 },
524 { "PDT", -7 * 3600 },
525 { "PST", -8 * 3600 },
526 };
527 const char *p;
528 long tzhour = 0, tzmin = 0;
529 size_t i;
530
531 for (; ISSPACE((unsigned char)*s); s++)
532 ;
533 switch (*s) {
534 case '-': /* offset */
535 case '+':
536 for (i = 0, p = s + 1; i < 2 && ISDIGIT((unsigned char)*p); i++, p++)
537 tzhour = (tzhour * 10) + (*p - '0');
538 if (*p == ':')
539 p++;
540 for (i = 0; i < 2 && ISDIGIT((unsigned char)*p); i++, p++)
541 tzmin = (tzmin * 10) + (*p - '0');
542 return ((tzhour * 3600) + (tzmin * 60)) * (s[0] == '-' ? -1 : 1);
543 default: /* timezone name */
544 for (i = 0; ISALPHA((unsigned char)s[i]); i++)
545 ;
546 if (i != 3)
547 return 0;
548 /* compare timezone and adjust offset relative to UTC */
549 for (i = 0; i < sizeof(tzones) / sizeof(*tzones); i++) {
550 if (!memcmp(s, tzones[i].name, 3))
551 return tzones[i].offhour;
552 }
553 }
554 return 0;
555 }
556
557 /* Parse time string `s` into the UNIX timestamp `tp`.
558 Returns 0 on success or -1 on failure. */
559 static int
560 parsetime(const char *s, long long *tp)
561 {
562 static const struct {
563 char *name;
564 int len;
565 } mons[] = {
566 { STRP("January"), },
567 { STRP("February"), },
568 { STRP("March"), },
569 { STRP("April"), },
570 { STRP("May"), },
571 { STRP("June"), },
572 { STRP("July"), },
573 { STRP("August"), },
574 { STRP("September"), },
575 { STRP("October"), },
576 { STRP("November"), },
577 { STRP("December"), },
578 };
579 int va[6] = { 0 }, i, j, v, vi;
580 size_t m;
581
582 for (; ISSPACE((unsigned char)*s); s++)
583 ;
584 if (!ISDIGIT((unsigned char)*s) && !ISALPHA((unsigned char)*s))
585 return -1;
586
587 if (ISDIGIT((unsigned char)s[0]) &&
588 ISDIGIT((unsigned char)s[1]) &&
589 ISDIGIT((unsigned char)s[2]) &&
590 ISDIGIT((unsigned char)s[3])) {
591 /* formats "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S" or "%Y%m%d%H%M%S" */
592 vi = 0;
593 } else {
594 /* format: "[%a, ]%d %b %Y %H:%M:%S" */
595 /* parse "[%a, ]%d %b %Y " part, then use time parsing as above */
596 for (; ISALPHA((unsigned char)*s); s++)
597 ;
598 for (; ISSPACE((unsigned char)*s); s++)
599 ;
600 if (*s == ',')
601 s++;
602 for (; ISSPACE((unsigned char)*s); s++)
603 ;
604 for (v = 0, i = 0; i < 2 && ISDIGIT((unsigned char)*s); s++, i++)
605 v = (v * 10) + (*s - '0');
606 va[2] = v; /* day */
607 for (; ISSPACE((unsigned char)*s); s++)
608 ;
609 /* end of word month */
610 for (j = 0; ISALPHA((unsigned char)s[j]); j++)
611 ;
612 /* check month name */
613 if (j < 3 || j > 9)
614 return -1; /* month cannot match */
615 for (m = 0; m < sizeof(mons) / sizeof(*mons); m++) {
616 /* abbreviation (3 length) or long name */
617 if ((j == 3 || j == mons[m].len) &&
618 !strncasecmp(mons[m].name, s, j)) {
619 va[1] = m + 1;
620 s += j;
621 break;
622 }
623 }
624 if (m >= 12)
625 return -1; /* no month found */
626 for (; ISSPACE((unsigned char)*s); s++)
627 ;
628 for (v = 0, i = 0; i < 4 && ISDIGIT((unsigned char)*s); s++, i++)
629 v = (v * 10) + (*s - '0');
630 /* obsolete short year: RFC 2822 4.3 */
631 if (i == 2 || i == 3)
632 v += (i == 2 && v >= 0 && v <= 49) ? 2000 : 1900;
633 va[0] = v; /* year */
634 for (; ISSPACE((unsigned char)*s); s++)
635 ;
636 /* parse only regular time part, see below */
637 vi = 3;
638 }
639
640 /* parse time parts (and possibly remaining date parts) */
641 for (; *s && vi < 6; vi++) {
642 for (i = 0, v = 0; i < ((vi == 0) ? 4 : 2) &&
643 ISDIGIT((unsigned char)*s); s++, i++) {
644 v = (v * 10) + (*s - '0');
645 }
646 va[vi] = v;
647
648 if ((vi < 2 && *s == '-') ||
649 (vi == 2 && (*s == 'T' || *s == 't' || ISSPACE((unsigned char)*s))) ||
650 (vi > 2 && *s == ':'))
651 s++;
652 }
653
654 /* skip milliseconds in for example: "%Y-%m-%dT%H:%M:%S.000Z" */
655 if (*s == '.') {
656 for (s++; ISDIGIT((unsigned char)*s); s++)
657 ;
658 }
659
660 /* invalid range */
661 if (va[0] < 0 || va[0] > 9999 ||
662 va[1] < 1 || va[1] > 12 ||
663 va[2] < 1 || va[2] > 31 ||
664 va[3] < 0 || va[3] > 23 ||
665 va[4] < 0 || va[4] > 59 ||
666 va[5] < 0 || va[5] > 60) /* allow leap second */
667 return -1;
668
669 *tp = datetounix(va[0] - 1900, va[1] - 1, va[2], va[3], va[4], va[5]) -
670 gettzoffset(s);
671
672 return 0;
673 }
674
675 static void
676 printfields(void)
677 {
678 string_print_timestamp(&ctx.fields[FeedFieldTime].str);
679 putchar(FieldSeparator);
680 string_print_trimmed(&ctx.fields[FeedFieldTitle].str);
681 putchar(FieldSeparator);
682 string_print_uri(&ctx.fields[FeedFieldLink].str);
683 putchar(FieldSeparator);
684 string_print_encoded(&ctx.fields[FeedFieldContent].str);
685 putchar(FieldSeparator);
686 fputs(contenttypes[ctx.contenttype], stdout);
687 putchar(FieldSeparator);
688 string_print_trimmed(&ctx.fields[FeedFieldId].str);
689 putchar(FieldSeparator);
690 string_print_trimmed(&ctx.fields[FeedFieldAuthor].str);
691 putchar(FieldSeparator);
692 string_print_uri(&ctx.fields[FeedFieldEnclosure].str);
693 putchar(FieldSeparator);
694 string_print_trimmed_multi(&ctx.fields[FeedFieldCategory].str);
695 putchar('\n');
696
697 if (ferror(stdout)) /* check for errors but do not flush */
698 checkfileerror(stdout, "<stdout>", 'w');
699 }
700
701 static int
702 istag(const char *name, size_t len, const char *name2, size_t len2)
703 {
704 return (len == len2 && !strcasecmp(name, name2));
705 }
706
707 static int
708 isattr(const char *name, size_t len, const char *name2, size_t len2)
709 {
710 return (len == len2 && !strcasecmp(name, name2));
711 }
712
713 static void
714 xmlattr(XMLParser *p, const char *t, size_t tl, const char *n, size_t nl,
715 const char *v, size_t vl)
716 {
717 /* handles transforming inline XML to data */
718 if (ISINCONTENT(ctx)) {
719 if (ctx.contenttype == ContentTypeHTML)
720 xmldata(p, v, vl);
721 return;
722 }
723
724 if (!ctx.tag.id)
725 return;
726
727 /* content-type may be for Atom: text, xhtml, html or a mime-type.
728 for MRSS (media:description): plain, html. */
729 if (ISCONTENTTAG(ctx)) {
730 if (isattr(n, nl, STRP("type")))
731 string_append(&attrtype, v, vl);
732 return;
733 }
734
735 if (ctx.feedtype == FeedTypeRSS) {
736 if (ctx.tag.id == RSSTagEnclosure &&
737 isattr(n, nl, STRP("url"))) {
738 string_append(&tmpstr, v, vl);
739 } else if (ctx.tag.id == RSSTagGuid &&
740 isattr(n, nl, STRP("ispermalink"))) {
741 string_append(&attrispermalink, v, vl);
742 }
743 } else if (ctx.feedtype == FeedTypeAtom) {
744 if (ctx.tag.id == AtomTagLink) {
745 if (isattr(n, nl, STRP("rel"))) {
746 string_append(&attrrel, v, vl);
747 } else if (isattr(n, nl, STRP("href"))) {
748 string_append(&tmpstr, v, vl);
749 }
750 } else if (ctx.tag.id == AtomTagCategory &&
751 isattr(n, nl, STRP("term"))) {
752 string_append(&tmpstr, v, vl);
753 }
754 }
755 }
756
757 static void
758 xmlattrentity(XMLParser *p, const char *t, size_t tl, const char *n, size_t nl,
759 const char *data, size_t datalen)
760 {
761 char buf[8];
762 int len;
763
764 /* handles transforming inline XML to data */
765 if (ISINCONTENT(ctx)) {
766 if (ctx.contenttype == ContentTypeHTML)
767 xmldata(p, data, datalen);
768 return;
769 }
770
771 if (!ctx.tag.id)
772 return;
773
774 /* try to translate entity, else just pass as data to
775 * xmlattr handler. */
776 if ((len = xml_entitytostr(data, buf, sizeof(buf))) > 0)
777 xmlattr(p, t, tl, n, nl, buf, (size_t)len);
778 else
779 xmlattr(p, t, tl, n, nl, data, datalen);
780 }
781
782 static void
783 xmlattrend(XMLParser *p, const char *t, size_t tl, const char *n, size_t nl)
784 {
785 if (ISINCONTENT(ctx)) {
786 if (ctx.contenttype == ContentTypeHTML) {
787 /* handles transforming inline XML to data */
788 xmldata(p, "\"", 1);
789 ctx.attrcount = 0;
790 }
791 return;
792 }
793 }
794
795 static void
796 xmlattrstart(XMLParser *p, const char *t, size_t tl, const char *n, size_t nl)
797 {
798 if (ISINCONTENT(ctx)) {
799 if (ctx.contenttype == ContentTypeHTML) {
800 /* handles transforming inline XML to data */
801 if (!ctx.attrcount)
802 xmldata(p, " ", 1);
803 ctx.attrcount++;
804 xmldata(p, n, nl);
805 xmldata(p, "=\"", 2);
806 }
807 return;
808 }
809
810 if (attrispermalink.len && isattr(n, nl, STRP("ispermalink")))
811 string_clear(&attrispermalink);
812 else if (attrrel.len && isattr(n, nl, STRP("rel")))
813 string_clear(&attrrel);
814 else if (attrtype.len && isattr(n, nl, STRP("type")))
815 string_clear(&attrtype);
816 else if (tmpstr.len &&
817 (isattr(n, nl, STRP("href")) ||
818 isattr(n, nl, STRP("term")) ||
819 isattr(n, nl, STRP("url"))))
820 string_clear(&tmpstr); /* use the last value for multiple attribute values */
821 }
822
823 static void
824 xmldata(XMLParser *p, const char *s, size_t len)
825 {
826 if (!ctx.field)
827 return;
828
829 if (ISFEEDFIELDMULTI(fieldmap[ctx.tag.id]))
830 string_append(&tmpstr, s, len);
831 else
832 string_append(ctx.field, s, len);
833 }
834
835 static void
836 xmldataentity(XMLParser *p, const char *data, size_t datalen)
837 {
838 char buf[8];
839 int len;
840
841 if (!ctx.field)
842 return;
843
844 /* try to translate entity, else just pass as data to
845 * xmldata handler. */
846 if ((len = xml_entitytostr(data, buf, sizeof(buf))) > 0)
847 xmldata(p, buf, (size_t)len);
848 else
849 xmldata(p, data, datalen);
850 }
851
852 static void
853 xmltagstart(XMLParser *p, const char *t, size_t tl)
854 {
855 const FeedTag *f;
856
857 if (ISINCONTENT(ctx)) {
858 if (ctx.contenttype == ContentTypeHTML) {
859 ctx.attrcount = 0;
860 xmldata(p, "<", 1);
861 xmldata(p, t, tl);
862 }
863 return;
864 }
865
866 /* start of RSS or Atom item / entry */
867 if (ctx.feedtype == FeedTypeNone) {
868 if (istag(t, tl, STRP("entry")))
869 ctx.feedtype = FeedTypeAtom;
870 else if (istag(t, tl, STRP("item")))
871 ctx.feedtype = FeedTypeRSS;
872 return;
873 }
874
875 /* field tagid already set or nested tags. */
876 if (ctx.tag.id) {
877 /* nested <author><name> for Atom */
878 if (ctx.tag.id == AtomTagAuthor &&
879 istag(t, tl, STRP("name"))) {
880 memcpy(&(ctx.tag), &atomtagauthorname, sizeof(ctx.tag));
881 } else {
882 return; /* other nested tags are not allowed: return */
883 }
884 }
885
886 /* in item */
887 if (ctx.tag.id == TagUnknown) {
888 if (!(f = gettag(ctx.feedtype, t, tl)))
889 f = ¬ag;
890 memcpy(&(ctx.tag), f, sizeof(ctx.tag));
891 }
892
893 ctx.iscontenttag = (fieldmap[ctx.tag.id] == FeedFieldContent);
894 string_clear(&attrispermalink);
895 string_clear(&attrrel);
896 string_clear(&attrtype);
897 }
898
899 static void
900 xmltagstartparsed(XMLParser *p, const char *t, size_t tl, int isshort)
901 {
902 enum TagId tagid;
903
904 if (ISINCONTENT(ctx)) {
905 if (ctx.contenttype == ContentTypeHTML) {
906 if (isshort)
907 xmldata(p, "/>", 2);
908 else
909 xmldata(p, ">", 1);
910 }
911 return;
912 }
913
914 /* set tag type based on its attribute value */
915 if (ctx.tag.id == RSSTagGuid) {
916 /* if empty the default is "true" */
917 if (!attrispermalink.len ||
918 isattr(attrispermalink.data, attrispermalink.len, STRP("true")))
919 ctx.tag.id = RSSTagGuidPermalinkTrue;
920 else
921 ctx.tag.id = RSSTagGuidPermalinkFalse;
922 } else if (ctx.tag.id == AtomTagLink) {
923 /* empty or "alternate": other types could be
924 "enclosure", "related", "self" or "via" */
925 if (!attrrel.len || isattr(attrrel.data, attrrel.len, STRP("alternate")))
926 ctx.tag.id = AtomTagLinkAlternate;
927 else if (isattr(attrrel.data, attrrel.len, STRP("enclosure")))
928 ctx.tag.id = AtomTagLinkEnclosure;
929 else
930 ctx.tag.id = AtomTagLink; /* unknown */
931 }
932
933 tagid = ctx.tag.id;
934
935 /* map tag type to field: unknown or lesser priority is ignored,
936 when tags of the same type are repeated only the first is used. */
937 if (fieldmap[tagid] == -1 ||
938 (!ISFEEDFIELDMULTI(fieldmap[tagid]) &&
939 tagid <= ctx.fields[fieldmap[tagid]].tagid)) {
940 return;
941 }
942
943 if (ctx.iscontenttag) {
944 ctx.iscontent = 1;
945 ctx.iscontenttag = 0;
946
947 /* detect content-type based on type attribute */
948 if (attrtype.len) {
949 if (isattr(attrtype.data, attrtype.len, STRP("html")) ||
950 isattr(attrtype.data, attrtype.len, STRP("xhtml")) ||
951 isattr(attrtype.data, attrtype.len, STRP("text/html")) ||
952 isattr(attrtype.data, attrtype.len, STRP("text/xhtml")) ||
953 isattr(attrtype.data, attrtype.len, STRP("application/xhtml+xml")))
954 ctx.contenttype = ContentTypeHTML;
955 else /* unknown: handle as base64 text data */
956 ctx.contenttype = ContentTypePlain;
957 } else {
958 /* default content-type */
959 if (tagid == RSSTagContentEncoded || tagid == RSSTagDescription)
960 ctx.contenttype = ContentTypeHTML;
961 else
962 ctx.contenttype = ContentTypePlain;
963 }
964 }
965
966 ctx.field = &(ctx.fields[fieldmap[tagid]].str);
967 ctx.fields[fieldmap[tagid]].tagid = tagid;
968
969 /* clear field if it is overwritten (with a priority order) for the new
970 value, if the field can have multiple values then do not clear it. */
971 if (!ISFEEDFIELDMULTI(fieldmap[ctx.tag.id]))
972 string_clear(ctx.field);
973 }
974
975 static void
976 xmltagend(XMLParser *p, const char *t, size_t tl, int isshort)
977 {
978 size_t i;
979
980 if (ctx.feedtype == FeedTypeNone)
981 return;
982
983 if (ISINCONTENT(ctx)) {
984 /* not a closed content field */
985 if (!istag(ctx.tag.name, ctx.tag.len, t, tl)) {
986 if (!isshort && ctx.contenttype == ContentTypeHTML) {
987 xmldata(p, "</", 2);
988 xmldata(p, t, tl);
989 xmldata(p, ">", 1);
990 }
991 return;
992 }
993 } else if (ctx.tag.id && istag(ctx.tag.name, ctx.tag.len, t, tl)) {
994 /* matched tag end: close it */
995 /* copy also to the link field if the attribute isPermaLink="true"
996 and it is not set by a tag with higher priority. */
997 if (ctx.tag.id == RSSTagGuidPermalinkTrue && ctx.field &&
998 ctx.tag.id > ctx.fields[FeedFieldLink].tagid) {
999 string_clear(&ctx.fields[FeedFieldLink].str);
1000 string_append(&ctx.fields[FeedFieldLink].str,
1001 ctx.field->data, ctx.field->len);
1002 ctx.fields[FeedFieldLink].tagid = ctx.tag.id;
1003 }
1004 } else if (!ctx.tag.id && ((ctx.feedtype == FeedTypeAtom &&
1005 istag(t, tl, STRP("entry"))) || /* Atom */
1006 (ctx.feedtype == FeedTypeRSS &&
1007 istag(t, tl, STRP("item"))))) /* RSS */
1008 {
1009 /* end of RSS or Atom entry / item */
1010 printfields();
1011
1012 /* clear strings */
1013 for (i = 0; i < FeedFieldLast; i++) {
1014 string_clear(&ctx.fields[i].str);
1015 ctx.fields[i].tagid = TagUnknown;
1016 }
1017 ctx.contenttype = ContentTypeNone;
1018 /* allow parsing of Atom and RSS concatenated in one XML stream. */
1019 ctx.feedtype = FeedTypeNone;
1020 } else {
1021 return; /* not end of field */
1022 }
1023
1024 /* temporary string: for fields that cannot be processed
1025 directly and need more context, for example by its tag
1026 attributes, like the Atom link rel="alternate|enclosure". */
1027 if (tmpstr.len && ctx.field) {
1028 if (ISFEEDFIELDMULTI(fieldmap[ctx.tag.id])) {
1029 if (ctx.field->len)
1030 string_append(ctx.field, FieldMultiSeparator, 1);
1031 string_append(ctx.field, tmpstr.data, tmpstr.len);
1032 } else {
1033 string_clear(ctx.field);
1034 string_append(ctx.field, tmpstr.data, tmpstr.len);
1035 }
1036 }
1037
1038 /* close field */
1039 string_clear(&tmpstr); /* reuse and clear temporary string */
1040
1041 if (ctx.tag.id == AtomTagAuthorName)
1042 memcpy(&(ctx.tag), &atomtagauthor, sizeof(ctx.tag)); /* outer tag */
1043 else
1044 memcpy(&(ctx.tag), ¬ag, sizeof(ctx.tag));
1045
1046 ctx.iscontent = 0;
1047 ctx.field = NULL;
1048 }
1049
1050 int
1051 main(int argc, char *argv[])
1052 {
1053 if (pledge("stdio", NULL) == -1)
1054 err(1, "pledge");
1055
1056 if (argc > 1) {
1057 if (uri_parse(argv[1], &baseuri) != -1 && baseuri.proto[0])
1058 baseurl = argv[1];
1059 else
1060 errx(1, "baseurl incorrect or too long");
1061 }
1062
1063 memcpy(&(ctx.tag), ¬ag, sizeof(ctx.tag));
1064
1065 parser.xmlattr = xmlattr;
1066 parser.xmlattrentity = xmlattrentity;
1067 parser.xmlattrend = xmlattrend;
1068 parser.xmlattrstart = xmlattrstart;
1069 parser.xmlcdata = xmldata;
1070 parser.xmldata = xmldata;
1071 parser.xmldataentity = xmldataentity;
1072 parser.xmltagend = xmltagend;
1073 parser.xmltagstart = xmltagstart;
1074 parser.xmltagstartparsed = xmltagstartparsed;
1075
1076 /* NOTE: GETNEXT is defined in xml.h for inline optimization */
1077 xml_parse(&parser);
1078
1079 checkfileerror(stdin, "<stdin>", 'r');
1080 checkfileerror(stdout, "<stdout>", 'w');
1081
1082 return 0;
1083 }