| #!/usr/bin/env ruby |
| |
| # DRAFT DRAFT DRAFT |
| # DRAFT DRAFT DRAFT |
| # DRAFT DRAFT DRAFT |
| # DRAFT DRAFT DRAFT |
| # DRAFT DRAFT DRAFT |
| |
| # |
| # ICLA PDF parsing support |
| # |
| # Try to extract user text from ICLA PDFs. |
| |
| # The Gem is not 100% accurate in creating a text version of the page. |
| # Also it's tricky to extract the text accurately. |
| |
| # So we try other methods first: |
| # - if there is a form, return its fields |
| # - if there are FreeText Annotations, return them in page order |
| # - use show_text_with_positioning as that seems to be used for PDF updates |
| # - where the PDF only uses show_text, the Gem is better at combining the data, so use page.txt |
| |
| require 'pdf-reader' |
| |
| # TODO perhaps always extract all the data types then choose the best |
| # Should turn hash values into arrays? |
| module ICLAParser |
| # Process page to extract text with positioning elements |
| # These are often used instead of providing form fields |
| class Receiver |
| SKIP = [ |
| # Short elements that are not user data |
| 'Individual Contributor', |
| 'License Agreement', |
| '("Agreement") V2.0', |
| "as \"Not a Contribution.\"", |
| "inaccurate in any respect.", |
| "for your records.", |
| "1. Definitions.", |
| "Contributions and such derivative works.", |
| "litigation is filed.", |
| "Contributions." |
| ] |
| |
| def initialize(fontdict) |
| @texts = [] # show_text_with_positioning |
| @lines = [] # show_text |
| @tfs = nil # text font and size |
| @fontdict = fontdict |
| end |
| |
| # Some PDFs use show_text() multiple times in a line |
| def begin_text_object |
| @textobj = [] |
| end |
| |
| def end_text_object |
| @lines << @textobj.join('') |
| end |
| |
| def set_text_font_and_size(*args) |
| @tfs=args |
| end |
| |
| def show_text(string) |
| font = @fontdict[@tfs.first] |
| utf8 = ICLAParser.string_to_utf8(string, font) |
| @textobj << utf8 |
| end |
| |
| def show_text_with_positioning(*args) |
| font = @fontdict[@tfs.first] |
| # args are Strings (in the current font encoding) interspersed with integer spacing adjustments; only want the strings |
| # We assume the positioning does not overlay characters so can be ignored |
| chars = [] |
| args.flatten.each do |arg| |
| if arg.is_a?(String) |
| char = ICLAParser.string_to_utf8(arg, font) |
| chars << char |
| end |
| end |
| val = chars.join("").strip |
| len = val.length |
| # some PDFs have the individual text in this format so skip long lines which are unlikely to be user data |
| # Could perhaps have full list of expected text lines instead. |
| unless len == 0 or len > 50 or SKIP.include? val |
| @texts << val |
| end |
| end |
| |
| def get_text |
| @texts |
| end |
| |
| def get_lines |
| @lines |
| end |
| |
| end |
| |
| def self.string_to_utf8(string, font) |
| chars = [] |
| glyphs = font.unpack(string) |
| glyphs.each do |glyph_code| |
| char = font.to_utf8(glyph_code) |
| # One pdf (yev) has spurious \t\r<sp>?<nbsp> translated from 36 => [9, 13, 32, 194, 160] |
| if glyph_code == 36 and char =~ /^\t\r / |
| char = ' ' |
| end |
| chars << char |
| end |
| chars.join('') |
| end |
| |
| # Standard form field names for other code to use |
| NAME2FIELD = { |
| 'fullname' => :FullName, |
| 'publicname' => :PublicName, |
| 'mailingaddress' => :MailingAddress, |
| 'mailingaddress2' => :MailingAddress2, |
| 'postaladdress' => :MailingAddress, |
| 'country' => :Country, |
| 'telephone' => :Telephone, |
| 'e-mail' => :EMail, |
| 'preferredapacheid(s)' => :ApacheID, |
| 'notifyproject' => :Project, |
| 'date' => :Date, |
| 'signature' => :Signature, |
| } |
| |
| # canonicalise the names found in the PDF |
| def self.canon_field_name(pdfname) |
| NAME2FIELD[pdfname.gsub(' ','').downcase] || pdfname |
| end |
| |
| # parse the PDF |
| def self.parse(path) |
| data=Hash.new |
| metadata = {} |
| data[:_meta] = metadata |
| metadata[:dataSource] = {} # have we found anything |
| freetext = {} # gather the free text details |
| debug={} |
| begin |
| reader = PDF::Reader.new(path) |
| %w(pdf_version info metadata page_count).each do |i| |
| metadata[i] = reader.public_send(i) |
| end |
| reader.objects.each do |k,v| |
| type = v[:Type] rescue nil |
| subtype = v[:Subtype] rescue nil |
| |
| if type == :Annot |
| if subtype == :FreeText # These are not directly associated with forms |
| rect = v[:Rect] |
| # rect can be a reference. If so, it seems there may be multiple copies with different IDs but same Rect coords and contents |
| if rect.is_a?(PDF::Reader::Reference) |
| rect = reader.objects.deref(rect) |
| end |
| if rect.is_a?(Array) |
| contents = v[:Contents] |
| if contents and contents.length > 0 and contents != "\x14" # ignore "\x14" == ASCII DC4 |
| # Entries may be duplicated, so use a hash to store them |
| id = rect.inspect+contents # if the rect and contents match, then they overwrite each other |
| freetext[id] = {Contents: contents.strip, x: rect[0], y: rect[1]} |
| metadata[:dataSource]['FreeText'] = true |
| end |
| else |
| puts "warn: #{contents} Rect is #{rect.class} in #{path}" |
| end |
| else |
| key = v[:T] |
| if key |
| val = v[:V] |
| # This is a hack; should really find the font def and use that |
| if val |
| debug["#{key}"] = v.inspect |
| if val.bytes[0..1] == [254,255] |
| val = val.encode('utf-8','utf-16').strip |
| else |
| begin |
| val = val.encode('utf-8').strip |
| rescue Encoding::UndefinedConversionError |
| val = val.encode('utf-8','iso-8859-1').strip |
| end |
| end |
| val.gsub!("\x7F",'') # Not sure where these originate |
| if val.length > 0 |
| data[canon_field_name(key)] = val.gsub("\x7F",'') # Not sure where these originate |
| end |
| metadata[:dataSource]['Form'] = true |
| end |
| end |
| end |
| else |
| end |
| end # objects |
| if freetext.size > 0 |
| data[:text] = [] |
| # Sort by Y descending (down the page) and X ascending (across) |
| # split into separate chunks if the difference in Y is more than a few points |
| how_close = 3 |
| freetext.values. # no need for ids any more |
| sort_by{|e| -e[:y] }. # sort by Y desc |
| slice_when{|i,j| (i[:y]-j[:y]) > how_close}. # gather nearby Y values in case there are multiple entries on a line |
| each do |k| |
| data[:text] << k. |
| sort_by{|l| l[:x]}. # sort by X ascending |
| map{|v| v[:Contents]}.join(", ") |
| end |
| end |
| if metadata[:dataSource].size == 0 or ((data[:text].size rescue 0) <= 1 and data.size < 3) # No annotations found or not useful |
| page1 = nil # cache for page 1 |
| fontdict = Hash.new |
| # Try looking for text sections instead |
| receiver = Receiver.new(fontdict) |
| reader.pages.each do |page| |
| # extract the fonts (needed for conversion to utf-8) |
| page.fonts.each do |label, font| |
| fontdict[label] ||= PDF::Reader::Font.new(page.objects, page.objects.deref(font)) |
| end |
| page.walk(receiver) |
| page1 ||= page.text |
| end |
| # pickup up the collected strings |
| text = receiver.get_text() |
| # p text |
| lines = receiver.get_lines() # do we still need these? |
| debug[:lines] = lines |
| if text.length > 3 |
| metadata[:dataSource]['Text'] = true |
| data[:text] = text |
| else |
| page1.each_line.slice_before(/^\s+Full name:/).each_with_index do |lump, i| |
| if i == 1 # starts with Full name |
| metadata[:dataSource]['Page'] = true |
| # drop the postamble |
| form = lump.slice_before(/^\S/).first |
| # split into headers |
| form.slice_before(/^\s+.+:/).each do |lines| |
| # trim leading and trailing blanks and underscores and drop blank lines |
| line = lines.map{|l| l.sub(/^[ _]+/,'').sub(/[ _]+$/,'')}.select{|l| l.length > 0}.join(',') |
| case line |
| when /^\s*(?:\(optional\) )?(.+):\s+(.*)/ |
| data[canon_field_name($1)] = $2 |
| else |
| data[:unmatched] ||= [] |
| data[:unmatched] << line |
| end |
| end |
| end |
| end |
| end |
| end |
| rescue Exception => e |
| data[:error]="Error processing #{path} => #{e.inspect} #{caller}" |
| end |
| # data[:debug] = debug |
| # TODO attempt to classify data[:text] items? |
| data |
| end |
| end |
| |
| if __FILE__ == $0 |
| require 'pp' |
| pp ICLAParser.parse(ARGV.first) |
| end |
