diff --git a/History.txt b/History.txt index b443b3a..59a1366 100644 --- a/History.txt +++ b/History.txt @@ -14,6 +14,10 @@ Unreleased * Enable frozen_string_literal in the generated parsers and stop mutating strings that Ruby 3.4+ will freeze (Symbol#to_s in Entry#digest, the name parser's word buffer); these emitted deprecation warnings and would raise FrozenError in a future Ruby * Allocate fewer objects when parsing: the lexer no longer builds a method name per token or copies matched punctuation, and the grammar drops redundant actions * Fix a memory leak in month parsing: Entry::MONTHS_FILTER cached every parsed month and never released it, so memory use and parse time grew with each parse (#196). The constant is removed; months are normalised by Entry#parse_month. Entries no longer share month Values; padded months like " Jan " are recognised, while "0" and "00" are no longer mistaken for months +* Accept every character BibTeX allows in identifiers in field names, @string names and macro references, e.g. x;y, p[1], a|b, a^b, a~b, a, a@b and @foo; these used to raise an unexpected-token error. An '@' followed by a name and '{' or '(' still starts a new entry, and a name that starts with '@' ends before the next '@' +* Bibliography#save_to (and #save) replace the file atomically, so a failure while serialising or writing leaves the existing file untouched instead of empty or partly written; symlinked targets and permission bits are kept, and devices, pipes and files in read-only directories are written directly +* Bibliography#save_to no longer adds :quotes to the options hash passed in +* Reimplement Entry's dynamic field writers, Bibliography#save_to and the lexer's identifier and key rules from a written specification, without the code they replace (#154) 6.2.0 / 2025-07-20 ================== diff --git a/features/identifiers.feature b/features/identifiers.feature new file mode 100644 index 0000000..eadfeef --- /dev/null +++ b/features/identifiers.feature @@ -0,0 +1,102 @@ +Feature: Identifier characters in field names, @string names and macro references + As a hacker who works with bibliographies + I want to parse every identifier that BibTeX itself accepts + Because exported bibliographies use field names like "a/b" or "p[1]" + + Scenario Outline: A field name containing () + When I parse the following file with options : + """ + @misc{key, = {value}, title = {Title}} + @misc{other, title = {Other}} + """ + Then my bibliography should contain 2 misc + And the entry with key "key" should have a field "" with the value "value" + And the entry with key "key" should have a field "title" with the value "Title" + + Examples: + | name | mode | options | + | a/b | strict | strict: true | + | x;y | strict | strict: true | + | p[1] | strict | strict: true | + | a\|b | strict | strict: true | + | a^b | strict | strict: true | + | a~b | strict | strict: true | + | a | strict | strict: true | + | a@b | strict | strict: true | + | a/b | non-strict | strict: false | + | x;y | non-strict | strict: false | + | p[1] | non-strict | strict: false | + | a\|b | non-strict | strict: false | + | a^b | non-strict | strict: false | + | a~b | non-strict | strict: false | + | a | non-strict | strict: false | + | a@b | non-strict | strict: false | + | a/b | strict with :strip_comments | strip_comments: true | + | a@b | strict with :strip_comments | strip_comments: true | + | p[1] | non-strict with :strip_comments | strict: false, strip_comments: true | + | a | non-strict with :strip_comments | strict: false, strip_comments: true | + + Scenario Outline: A @string named and a reference to it () + When I parse the following file with options : + """ + @string{ = "Journal of Examples" } + @article{key, journal = , title = {Title}} + """ + And I replace all strings in my bibliography + Then the string "" should be "Journal of Examples" + And the entry with key "key" should have a field "journal" with the value "Journal of Examples" + + Examples: + | name | mode | options | + | a/b | strict | strict: true | + | x;y | strict | strict: true | + | p[1] | strict | strict: true | + | a\|b | strict | strict: true | + | a^b | strict | strict: true | + | a~b | strict | strict: true | + | a | strict | strict: true | + | a@b | strict | strict: true | + | a@b | non-strict | strict: false | + | x;y | strict with :strip_comments | strip_comments: true | + + Scenario: Percent signs in names depend on :strip_comments + When I parse the following file with options strict: true: + """ + @string{ a/b%c = "with percent" } + @misc{key, x%y = a/b%c} + """ + And I replace all strings in my bibliography + Then the entry with key "key" should have a field "x%y" with the value "with percent" + + Scenario: Percent signs start a comment with :strip_comments + When I parse the following file with options strip_comments: true: + """ + @string{ a/b = "no percent" } % a comment + @misc{key, + x;y = a/b % a comment + } + """ + And I replace all strings in my bibliography + Then the entry with key "key" should have a field "x;y" with the value "no percent" + + Scenario: Field names made of several words in non-strict mode + When I parse the following file with options strict: false: + """ + @misc{key, Early/Access Date[1] = {NOV}} + """ + Then the entry with key "key" should have a field "early/access_date[1]" with the value "NOV" + + Scenario Outline: Keys containing '/' with + When I parse the following file with options : + """ + @article{DBLP:journals/dq/RossinK99, title = {Data Errors}} + """ + Then my bibliography should contain an article with id "DBLP:journals/dq/RossinK99" + + Examples: + | options | + | strict: true | + | strict: false | + | strip_comments: true | + | allow_missing_keys: true | + | strip_comments: true, allow_missing_keys: true | diff --git a/features/step_definitions/bibtex_steps.rb b/features/step_definitions/bibtex_steps.rb index 53d6fc4..49a1dea 100644 --- a/features/step_definitions/bibtex_steps.rb +++ b/features/step_definitions/bibtex_steps.rb @@ -8,6 +8,16 @@ @bibliography = BibTeX.parse(string) end +# Options are written as "name: value" pairs separated by commas, e.g. +# "strict: false, strip_comments: true"; values are true or false. +When(/^I parse the following file with options (.+):$/) do |options, string| + options = options.split(/\s*,\s*/).to_h do |pair| + name, value = pair.split(/\s*:\s*/, 2) + [name.to_sym, value == 'true'] + end + @bibliography = BibTeX.parse(string, options) +end + When(/^I search for "([^"]*)"$/) do |query| @result = @bibliography.query(query) end diff --git a/lib/bibtex/bibliography.rb b/lib/bibtex/bibliography.rb index d4ba371..ff2353b 100644 --- a/lib/bibtex/bibliography.rb +++ b/lib/bibtex/bibliography.rb @@ -18,6 +18,9 @@ # along with this program. If not, see . #++ +require 'fileutils' +require 'tempfile' + module BibTeX # # The Bibliography class models a BibTeX bibliography; @@ -137,14 +140,24 @@ def save(options = {}) save_to(@path, options) end - # Saves the bibliography to a file at the given path. Returns the bibliography. + # Saves the bibliography to a file at the given path (a String or a + # Pathname), replacing a regular file atomically where it can and + # writing directly otherwise. The options are passed on to #to_s; + # values are enclosed in braces unless :quotes says otherwise. + # Returns the bibliography. def save_to(path, options = {}) - options[:quotes] ||= %w[{ }] - - File.open(path, 'w:UTF-8') do |f| - f.write(to_s(options)) + # Serialise before touching the file system, so an error in #to_s + # leaves no trace. A nil :quotes means the default, as an absent key + # does. Always UTF-8, whatever Encoding.default_external is, so the + # file reads back through BibTeX.open. + content = to_s(options.merge(quotes: options[:quotes] || %w[{ }])).encode(Encoding::UTF_8) + # Replace the file a symlink points to, not the link. + target = File.exist?(path) ? File.realpath(path) : path + if replace_atomically?(target) + write_atomically(target, content) + else + File.write(target, content, encoding: Encoding::UTF_8) end - self end @@ -568,6 +581,65 @@ def uniq(*_arguments) private + # A short, fixed prefix: a name derived from the target's could exceed + # the file system's limit for long target names. + TEMP_NAME = ['.bibtex-save', '.tmp'].freeze + private_constant :TEMP_NAME + + # Whether #save_to may replace target atomically: only a regular file + # (or a missing one) that may be written, in a directory where the + # temporary file can be created. Anything else -- a read-only file, + # a read-only directory, a device, FIFO or socket, a dangling symlink -- + # is written directly, so the OS behaves and fails as for File.write. + def replace_atomically?(target) + return false unless File.writable?(File.dirname(target)) + + stat = File.lstat(target) + stat.file? && stat.writable? + rescue Errno::ENOENT + true + end + + # Writes content to a temporary file in the target's directory (so the + # rename stays on one file system), syncs it and renames it over the + # target: a failed write never leaves a partial file. Renaming over an + # existing file is atomic on POSIX; on Windows File.rename replaces the + # target but is not guaranteed to be atomic. The directory is not + # fsynced, so the rename itself may not survive a crash; IO#fsync can + # raise NotImplementedError on some platforms, and is skipped there. + # + # The file keeps its permission bits (without setuid, setgid and sticky) + # and a new one gets 0666 minus the umask rather than the temporary + # file's 0600. Hard links are out of scope: the target gets a new inode, + # so other links to the old file keep the old content (as do its owner, + # ACLs and xattrs). Text mode, as for File.write, keeps the platform's + # newline conversion. + def write_atomically(target, content) + mode = File.exist?(target) ? File.stat(target).mode & 0o777 : 0o666 & ~File.umask + temp = Tempfile.create(TEMP_NAME, File.dirname(target), encoding: Encoding::UTF_8) + begin + temp.write(content) + temp.flush + fsync(temp) + temp.chmod(mode) + temp.close + File.rename(temp.path, target) + temp = nil + ensure + if temp + temp.close + FileUtils.rm_f(temp.path) + end + end + end + + def fsync(file) + file.fsync + rescue NotImplementedError + # No fsync on this platform; the data has still been flushed. + nil + end + def query_handler(selector) case selector.to_s when /first|distinct|detect/i diff --git a/lib/bibtex/entry.rb b/lib/bibtex/entry.rb index 0cda168..f0fe30b 100644 --- a/lib/bibtex/entry.rb +++ b/lib/bibtex/entry.rb @@ -259,13 +259,21 @@ def inherited_fields names end - def method_missing(name, *args, &) + # Ghost methods: a writer `=` for any non-empty field name that + # does not itself end in '=' (so :'=', :'==' and :'a==' stay unknown + # methods), and the convert_, convert_! and + # from_ conversions. + FIELD_WRITER = /\A(?.*[^=])=\z/m + CONVERSION = /\A(?:convert|from)_(?[a-z]+)(?!)?\z/ + private_constant :FIELD_WRITER, :CONVERSION + + def method_missing(name, *, &) if fields.key?(name) fields[name] - elsif name.to_s =~ /^(.+)=$/ - send(:add, Regexp.last_match(1).to_sym, args[0]) - elsif name =~ /^(?:convert|from)_([a-z]+)(!)?$/ - Regexp.last_match(2) ? convert!(Regexp.last_match(1), &) : convert(Regexp.last_match(1), &) + elsif (writer = FIELD_WRITER.match(name)) + write_field(writer[:field], *) + elsif (conversion = CONVERSION.match(name)) + conversion[:bang] ? convert!(conversion[:filter], &) : convert(conversion[:filter], &) elsif has_parent? && parent.provides?(name) parent.provide(name) else @@ -274,8 +282,9 @@ def method_missing(name, *args, &) end def respond_to_missing?(method, include_private = false) - provides?(method.to_sym) || method.to_s.match(/=$/) || - method =~ /^(?:convert|from)_([a-z]+)(!)?$/ || + provides?(method.to_sym) || + FIELD_WRITER.match?(method) || + CONVERSION.match?(method) || (has_parent? && parent.respond_to?(method, include_private)) || super end @@ -692,6 +701,14 @@ def normalize_month(month) end end + # Backs the ghost writers; checks the arity a real writer would have. + def write_field(name, *values) + raise ::ArgumentError, "wrong number of arguments (given #{values.length}, expected 1)" unless values.length == 1 + + # Same conversion as #add, like the generated writers above. + add(name, values[0]) + end + # Escapes every & not already preceded by a backslash, unless the field # is verbatim, a BibDesk URL field, or a howpublished containing \url. # diff --git a/lib/bibtex/lexer.rb b/lib/bibtex/lexer.rb index 1302d0b..ac1fa0d 100644 --- a/lib/bibtex/lexer.rb +++ b/lib/bibtex/lexer.rb @@ -40,6 +40,67 @@ class Lexer strip_comments: false } + # bibtex.web's id_class: an identifier (field, @string or macro name) + # may hold any printable ASCII character except white space and these + # (illegal_id_char). + ILLEGAL_ID_CHARS = %q{"#%'(),={}} + + # Characters of a citation key: those accepted before, plus '/' as in + # DBLP keys (DBLP:journals/dq/RossinK99). Braces only with a key that + # may not be empty. '-' comes last as it is spliced into a class. + KEY_CHAR = %q([:alnum:] :_!$?.+;&*'"/-) + private_constant :ILLEGAL_ID_CHARS, :KEY_CHAR + + class << self + private + + # A character class for identifiers. Non-ASCII letters and digits are + # a project extension, as is '%' unless :strip_comments makes it start + # a comment. + def id_char(percent:, at: true) + illegal = percent ? ILLEGAL_ID_CHARS.delete('%') : ILLEGAL_ID_CHARS + illegal += '@' unless at + # Escaped because the list is spliced into a character class, where + # ']', '\\', '^' or '-' would otherwise change its meaning. + "[[:alnum:]!-~&&[^#{Regexp.escape(illegal)}]]" + end + + # One word of a name. id_class allows '@' anywhere, but '@' also + # starts the next object, which error recovery after an unterminated + # object relies on. Identifiers cannot contain = , } # { or (, so the + # character after '@word' decides unambiguously: a name if (after + # white space, and TeX comments when they are stripped) it is one of + # = , } #, an object start otherwise, e.g. at '{' or '('. + # + # A word that starts with '@' ends before the next '@'. Otherwise the + # lookahead would be retried from every '@' of a long run like + # @a@a@a... and rescan the rest of it each time, which is quadratic. + # So @a@b = x is not a name; a@b and a@b@c still are. + def word_pattern(percent:) + gap = percent ? '\s*' : '(?:\s|%[^\n]*)*' + char = id_char(percent:) + "(?:(?!@)#{char}+|@#{id_char(percent:, at: false)}+(?=#{gap}[=,}#]))" + end + + # A name must not start with a digit (bibtex.web). + def name_pattern(percent:) + /(?![[:digit:]])#{word_pattern(percent:)}/ + end + + # Several words followed by '=', e.g. Web of Science's `Early Access + # Date'. The words are joined into one name; as before, a word + # may start with a digit. By the rule above only the last word, which + # is followed by '=', can start with '@'. + def spaced_name_pattern(percent:) + word = word_pattern(percent:) + /#{word}(?:[\t ]+#{word})+(?=\s*=)/ + end + + def key_pattern(extra, optional:) + /\s*[#{extra}#{KEY_CHAR}]#{optional ? '*' : '+'},/ + end + end + # Patterns Cache (#37: MacRuby does not cache regular expressions) @patterns = { space: /\s+/o, @@ -49,12 +110,10 @@ class Lexer eq: /\s*=\s*/o, comma: /\s*,\s*/o, number: /[[:digit:]]+/o, - name: %r{[[:alpha:][:digit:]/:_!$?.%+&*-]+}io, - # Several space-separated words followed by `=', e.g. the - # `Early Access Date' fields exported by Web of Science. - spaced_name: %r{[[:alpha:][:digit:]/:_!$?.%+&*-]+(?:[\t ]+[[:alpha:][:digit:]/:_!$?.%+&*-]+)+(?=\s*=)}io, - spaced_name_without_percent: %r{[[:alpha:][:digit:]/:_!$?.+&*-]+(?:[\t ]+[[:alpha:][:digit:]/:_!$?.+&*-]+)+(?=\s*=)}io, - name_without_percent: %r{[[:alpha:][:digit:]/:_!$?.+&*-]+}io, + name: name_pattern(percent: true), + name_without_percent: name_pattern(percent: false), + spaced_name: spaced_name_pattern(percent: true), + spaced_name_without_percent: spaced_name_pattern(percent: false), tex_comment: /%[^\n]*/o, tex_comments: /(?:\s*%[^\n]*)+/o, quote: /\s*"/o, @@ -68,10 +127,10 @@ class Lexer string: /string/io, comment: /comment\b/io, preamble: /preamble\b/io, - key: %r{\s*[[:alpha:][:digit:] /:_!$?.%+;&*'"{}-]+,}io, - optional_key: %r{\s*[[:alpha:][:digit:] /:_!$?.%+;&*'"-]*,}io, - key_without_percent: %r{\s*[[:alpha:][:digit:] /:_!$?.+;&*'"{}-]+,}io, - optional_key_without_percent: %r{\s*[[:alpha:][:digit:] /:_!$?.+;&*'"-]*,}io + key: key_pattern('%{}', optional: false), + optional_key: key_pattern('%', optional: true), + key_without_percent: key_pattern('{}', optional: false), + optional_key_without_percent: key_pattern('', optional: true) } MODE = Hash.new(:meta).merge( diff --git a/test/bibtex/test_bibliography_save.rb b/test/bibtex/test_bibliography_save.rb new file mode 100644 index 0000000..2a1fa07 --- /dev/null +++ b/test/bibtex/test_bibliography_save.rb @@ -0,0 +1,395 @@ +# frozen_string_literal: true + +require 'helper' +require 'pathname' +require 'tmpdir' + +module BibTeX + # BibTeX::Bibliography#save_to and #save. + class BibliographySaveTest < Minitest::Spec + SOURCE = <<~BIBTEX + @book{melville, + author = {Melville, Herman}, + title = {Moby-Dick; or, The Whale}, + year = 1851 + } + @book{goethe, + author = {von Goethe, Johann Wolfgang}, + title = {Die Leiden des jungen Werthers: Über Lotte, Ärger und Liebe}, + address = {Leipzig – Weygand}, + year = 1774 + } + BIBTEX + + describe 'saving' do + before do + @dir = Dir.mktmpdir('bibtex-save') + @path = File.join(@dir, 'out.bib') + @bib = BibTeX.parse(SOURCE) + end + + after { FileUtils.remove_entry(@dir) } + + def spy_on_to_s(bib) + calls = [] + original = bib.method(:to_s) + bib.define_singleton_method(:to_s) do |options = {}| + calls << options + original.call(options) + end + calls + end + + describe '#save_to' do + it 'writes to_s to the path' do + @bib.save_to(@path) + + assert_equal @bib.to_s(quotes: %w[{ }]), File.read(@path, encoding: 'utf-8') + end + + it 'returns the bibliography' do + assert_same @bib, @bib.save_to(@path) + end + + it 'uses { } as the default :quotes' do + calls = spy_on_to_s(@bib) + @bib.save_to(@path) + + assert_equal 1, calls.length + assert_equal %w[{ }], [*calls[0][:quotes]] + end + + it 'uses { } when :quotes is nil, without changing the caller\'s options' do + calls = spy_on_to_s(@bib) + options = { quotes: nil } + @bib.save_to(@path, options) + + assert_equal %w[{ }], [*calls[0][:quotes]] + assert_equal({ quotes: nil }, options) + assert_includes File.read(@path, encoding: 'utf-8'), 'title = {Moby-Dick; or, The Whale}' + end + + it 'passes its options on to to_s' do + calls = spy_on_to_s(@bib) + @bib.save_to(@path, quotes: '"', custom: :option) + + assert_equal '"', calls[0][:quotes] + assert_equal :option, calls[0][:custom] + end + + it 'honours the :quotes option' do + @bib.save_to(@path, quotes: '"') + content = File.read(@path, encoding: 'utf-8') + + assert_equal @bib.to_s(quotes: '"'), content + assert_includes content, 'title = "Moby-Dick; or, The Whale"' + end + + it 'writes braces by default' do + @bib.save_to(@path) + + assert_includes File.read(@path, encoding: 'utf-8'), 'title = {Moby-Dick; or, The Whale}' + end + + it 'writes UTF-8' do + @bib.save_to(@path) + bytes = File.binread(@path) + + assert_predicate bytes.dup.force_encoding(Encoding::UTF_8), :valid_encoding? + assert_includes bytes, 'Über'.b + assert_includes bytes, '–'.b + end + + it 'writes UTF-8 regardless of the default external encoding' do + original = Encoding.default_external + begin + Encoding.default_external = Encoding::ISO_8859_1 + @bib.save_to(@path) + ensure + Encoding.default_external = original + end + + # Read back in text mode: on Windows the file has CRLF line endings, + # like any text-mode write, and reading in text mode undoes that. + assert_equal @bib.to_s, File.read(@path, encoding: Encoding::UTF_8) + assert_includes File.binread(@path), 'Über'.b + end + + it 'round-trips non-ASCII content through BibTeX.open' do + @bib.save_to(@path) + reopened = BibTeX.open(@path) + + assert_equal 'Die Leiden des jungen Werthers: Über Lotte, Ärger und Liebe', reopened['goethe'].title.to_s + assert_equal 'Leipzig – Weygand', reopened['goethe'].address.to_s + assert_equal @bib.to_s, reopened.to_s + end + + it 'overwrites an existing file completely' do + File.write(@path, "#{'% old content that is much longer than the new one\n' * 200}@misc{old, title = {Old}}\n") + BibTeX.parse('@misc{new, title = {New}}').save_to(@path) + + assert_equal BibTeX.parse('@misc{new, title = {New}}').to_s, File.read(@path, encoding: 'utf-8') + assert_equal %w[new], BibTeX.open(@path).entries.keys + end + + it 'accepts a Pathname' do + result = @bib.save_to(Pathname.new(@path)) + + assert_same @bib, result + assert_equal @bib.to_s, File.read(@path, encoding: 'utf-8') + end + + it 'accepts a String' do + @bib.save_to(@path.to_s) + + assert_equal %w[melville goethe], BibTeX.open(@path).entries.keys + end + + it 'writes an empty bibliography as an empty file' do + File.write(@path, 'old') + Bibliography.new.save_to(@path) + + assert_equal '', File.read(@path) + end + + describe 'atomic replacement' do + # Makes every file Tempfile.create returns raise error in #fsync + # (by default as a full disk would) for the duration of the block. + def with_failing_fsync(error = IOError.new('disk full')) + original = Tempfile.method(:create) + Tempfile.define_singleton_method(:create) do |*args, **kwargs| + original.call(*args, **kwargs).tap do |file| + file.define_singleton_method(:fsync) { raise error } + end + end + yield + ensure + Tempfile.define_singleton_method(:create, original) + end + + def skip_unless_posix_permissions + skip 'needs POSIX permissions' if Gem.win_platform? + skip 'root ignores permission bits' if Process.uid.zero? + end + + it 'skips fsync where the platform does not implement it' do + with_failing_fsync(NotImplementedError.new('fsync')) { @bib.save_to(@path) } + + assert_equal @bib.to_s, File.read(@path, encoding: 'utf-8') + assert_equal %w[out.bib], Dir.children(@dir) + end + + it 'raises EACCES for a read-only file and leaves it as it was' do + skip_unless_posix_permissions + File.write(@path, 'old') + File.chmod(0o444, @path) + + assert_raises(Errno::EACCES) { @bib.save_to(@path) } + assert_equal 'old', File.read(@path) + assert_equal 0o444, File.stat(@path).mode & 0o7777 + end + + it 'writes a writable file in a read-only directory directly' do + skip_unless_posix_permissions + File.write(@path, 'old') + File.chmod(0o555, @dir) + begin + @bib.save_to(@path) + ensure + File.chmod(0o755, @dir) + end + + assert_equal @bib.to_s, File.read(@path, encoding: 'utf-8') + end + + it 'writes to a FIFO without replacing it' do + skip 'needs File.mkfifo' if Gem.win_platform? || !File.respond_to?(:mkfifo) + fifo = File.join(@dir, 'pipe.bib') + File.mkfifo(fifo) + reader = Thread.new { File.read(fifo, encoding: 'utf-8') } + begin + @bib.save_to(fifo) + + assert reader.join(5), 'the FIFO was not written to' + assert_equal @bib.to_s, reader.value + assert_equal 'fifo', File.ftype(fifo) + assert_equal %w[pipe.bib], Dir.children(@dir) + ensure + reader.kill + end + end + + it 'creates the file a dangling symlink points to' do + skip 'needs symlinks' if Gem.win_platform? + link = File.join(@dir, 'link.bib') + File.symlink('out.bib', link) + @bib.save_to(link) + + assert File.symlink?(link) + assert_equal @bib.to_s, File.read(@path, encoding: 'utf-8') + end + + it 'handles a target with a 254-byte name' do + name = "#{'a' * 250}.bib" + path = File.join(@dir, name) + File.write(path, 'old') + @bib.save_to(path) + + assert_equal @bib.to_s, File.read(path, encoding: 'utf-8') + assert_equal [name], Dir.children(@dir) + end + + it 'writes the same bytes as a text-mode File.write' do + reference = File.join(@dir, 'reference.bib') + File.write(reference, @bib.to_s, encoding: Encoding::UTF_8) + @bib.save_to(@path) + + assert_equal File.binread(reference), File.binread(@path) + @bib.save_to(@path) + + assert_equal File.binread(reference), File.binread(@path) + end + + it 'does not copy setuid, setgid or sticky bits' do + skip 'needs POSIX permissions' if Gem.win_platform? + File.write(@path, 'old') + File.chmod(0o4755, @path) + skip 'the file system dropped the setuid bit' unless File.stat(@path).mode & 0o7777 == 0o4755 + @bib.save_to(@path) + + assert_equal 0o755, File.stat(@path).mode & 0o7777 + end + + it 'leaves the original file and no temp file behind when the write fails' do + original = "@misc{keep, title = {Keep me}}\n" + File.write(@path, original) + + error = assert_raises(IOError) { with_failing_fsync { @bib.save_to(@path) } } + + assert_equal 'disk full', error.message + assert_equal original, File.read(@path) + assert_equal %w[out.bib], Dir.children(@dir) + end + + it 'does not create the file when the write fails' do + assert_raises(IOError) { with_failing_fsync { @bib.save_to(@path) } } + + assert_empty Dir.children(@dir) + end + + it 'leaves no temp file behind after a successful save' do + @bib.save_to(@path) + + assert_equal %w[out.bib], Dir.children(@dir) + end + + it 'creates a missing file with 0666 minus the umask' do + skip 'POSIX permissions' if Gem.win_platform? + @bib.save_to(@path) + + assert_equal 0o666 & ~File.umask, File.stat(@path).mode & 0o7777 + assert_equal @bib.to_s, File.read(@path, encoding: 'utf-8') + end + + it 'keeps the permission bits of an existing file' do + skip 'POSIX permissions' if Gem.win_platform? + File.write(@path, 'old') + File.chmod(0o640, @path) + @bib.save_to(Pathname.new(@path)) + + assert_equal 0o640, File.stat(@path).mode & 0o7777 + end + + it 'replaces the file a symlink points to, not the link' do + skip 'symlinks' if Gem.win_platform? + File.write(@path, 'old') + link = File.join(@dir, 'link.bib') + File.symlink('out.bib', link) + @bib.save_to(link) + + assert File.symlink?(link) + assert_equal @bib.to_s, File.read(@path, encoding: 'utf-8') + assert_equal %w[link.bib out.bib], Dir.children(@dir).sort + end + end + + describe 'when serialisation raises' do + before do + @original = "@misc{keep, title = {Keep me}}\n" + File.write(@path, @original) + @bib.define_singleton_method(:to_s) { |_options = {}| raise ArgumentError, 'boom' } + end + + it 'propagates the error' do + error = assert_raises(ArgumentError) { @bib.save_to(@path) } + + assert_equal 'boom', error.message + end + + it 'leaves the existing file untouched' do + assert_raises(ArgumentError) { @bib.save_to(@path) } + + assert_equal @original, File.read(@path) + end + + it 'leaves the existing file untouched when an element fails' do + bib = BibTeX.parse(SOURCE) + bib['goethe'].define_singleton_method(:to_s) { |_options = {}| raise ArgumentError, 'element boom' } + + assert_raises(ArgumentError) { bib.save_to(@path) } + assert_equal @original, File.read(@path) + end + + it 'does not create a missing file' do + missing = File.join(@dir, 'missing.bib') + + assert_raises(ArgumentError) { @bib.save_to(missing) } + refute_path_exists missing + end + end + end + + describe '#save' do + before do + File.write(@path, SOURCE) + @opened = BibTeX.open(@path) + end + + it 'writes back to the path the bibliography was opened from' do + @opened.delete(:melville) + @opened.save + + assert_equal %w[goethe], BibTeX.open(@path).entries.keys + end + + it 'returns the bibliography' do + assert_same @opened, @opened.save + end + + it 'passes its options on' do + @opened.save(quotes: '"') + + assert_includes File.read(@path, encoding: 'utf-8'), 'title = "Moby-Dick; or, The Whale"' + end + end + + describe 'BibTeX.open with a block' do + it 'saves the changes to the opened file' do + File.write(@path, SOURCE) + BibTeX.open(@path) { |bib| bib.delete(:goethe) } + + assert_equal %w[melville], BibTeX.open(@path).entries.keys + end + + it 'saves to the :out path instead when given' do + File.write(@path, SOURCE) + out = File.join(@dir, 'copy.bib') + BibTeX.open(@path, out: out) { |bib| bib.delete(:goethe) } + + assert_equal %w[melville], BibTeX.open(out).entries.keys + assert_equal SOURCE, File.read(@path, encoding: 'utf-8') + end + end + end + end +end diff --git a/test/bibtex/test_entry_writers.rb b/test/bibtex/test_entry_writers.rb new file mode 100644 index 0000000..f60b258 --- /dev/null +++ b/test/bibtex/test_entry_writers.rb @@ -0,0 +1,265 @@ +# frozen_string_literal: true + +require 'helper' + +module BibTeX + # Dynamic field writers on BibTeX::Entry. + class EntryWritersTest < Minitest::Spec + describe 'dynamic field writers' do + before do + @entry = Entry.new(bibtex_type: :book, bibtex_key: :moby, title: 'Moby Dick', keywords: 'whale') + end + + describe 'entry.foo = value' do + it 'adds a new field' do + @entry.isbn = '978-0142437247' + + assert_includes @entry.fields.keys, :isbn + assert_equal '978-0142437247', @entry[:isbn].to_s + end + + it 'adds a field with a name that has no reader yet' do + @entry.foo = 'bar' + + assert_equal 'bar', @entry.foo.to_s + end + + it 'replaces the value of an existing field' do + @entry.keywords = 'sea' + + assert_equal 'sea', @entry.keywords.to_s + assert_equal %i[title keywords], @entry.fields.keys + end + + it 'stores the value as a BibTeX::Value, like #add' do + @entry.edition = 1851 + + assert_kind_of Value, @entry[:edition] + assert_equal '1851', @entry[:edition].to_s + end + + it 'converts values exactly as #add does' do + ['Herman Melville', 1851, Value.new('x'), Names.new(Name.new(first: 'Herman', last: 'Melville'))].each do |value| + via_add = Entry.new(bibtex_type: :misc).add(:abstract, value) + via_writer = Entry.new(bibtex_type: :misc) + via_writer.abstract = value + + assert_equal via_add[:abstract].class, via_writer[:abstract].class, "class for #{value.inspect}" + assert_equal via_add[:abstract].to_s, via_writer[:abstract].to_s, "value for #{value.inspect}" + end + end + + it 'preserves BibTeX::Names (and other subclasses of BibTeX::Value)' do + @entry.narrator = Names.new(Name.new(first: 'Herman', last: 'Melville')) + + assert_instance_of Names, @entry[:narrator] + end + + it 'does not share the assigned Value object, like #add' do + value = Value.new('shared') + @entry.abstract = value + + refute_same value, @entry[:abstract] + end + + it 'works through send and public_send with Symbol and String names' do + @entry.send(:abstract=, 'a') + + assert_equal 'a', @entry.abstract.to_s + + writer = 'abstract=' + @entry.public_send(writer, 'b') + + assert_equal 'b', @entry.abstract.to_s + end + + it 'works for entries built with a block' do + entry = Entry.new do |e| + e.type = :book + e.key = :key + e.address = 'New York' + end + + assert_equal 'New York', entry.address.to_s + end + + it 'makes the new field part of the BibTeX output' do + @entry.isbn = '978-0142437247' + + assert_match(/isbn = \{978-0142437247\}/, @entry.to_s) + end + + it 'sets the field on a cross-referencing entry without touching the parent' do + bib = Bibliography.parse('@book{a, abstract = "A"} @incollection{a1, crossref = "a"}') + + assert_equal 'A', bib['a1'].abstract.to_s + + bib['a1'].abstract = 'Child' + + assert_equal 'Child', bib['a1'].abstract.to_s + assert_equal 'A', bib['a'].abstract.to_s + assert bib['a1'].field?(:abstract) + end + end + + describe '#respond_to? for writers' do + it 'returns exactly true for a Symbol writer name' do + assert_same true, @entry.respond_to?(:foo=) + end + + it 'returns exactly true for a String writer name' do + writer = 'foo=' + + assert_same true, @entry.respond_to?(writer) + end + + it 'returns exactly true for the writer of an existing field' do + writer = 'keywords=' + + assert_same true, @entry.respond_to?(:keywords=) + assert_same true, @entry.respond_to?(writer) + end + + it 'returns exactly true on an empty entry' do + assert_same true, Entry.new.respond_to?(:anything=) + end + + it 'exposes the writer through #method' do + @entry.method(:abstract=).call('via method') + + assert_equal 'via method', @entry.abstract.to_s + end + end + + describe 'a method name made only of "="' do + it 'is not a writer according to respond_to? (Symbol)' do + refute_respond_to @entry, :'=' + end + + it 'is not a writer according to respond_to? (String)' do + name = '=' + + refute_respond_to @entry, name + end + + it 'raises NoMethodError when called' do + assert_raises(NoMethodError) { @entry.send(:'=', 'x') } + name = '=' + + assert_raises(NoMethodError) { @entry.public_send(name, 'x') } + end + + it 'does not add a field' do + assert_raises(NoMethodError) { @entry.send(:'=', 'x') } + + assert_equal %i[title keywords], @entry.fields.keys + end + end + + describe "a writer name whose field part ends in '=' (a==)" do + it 'is not a writer' do + refute_respond_to @entry, :'a==' + assert_raises(NoMethodError) { @entry.send(:'a==', 'x') } + assert_equal %i[title keywords], @entry.fields.keys + end + end + + describe 'unknown methods' do + it 'still raise NoMethodError for undefined readers' do + assert_raises(NoMethodError) { @entry.undefined_field } + end + + it 'still report undefined readers as not responding' do + refute_respond_to @entry, :undefined_field + end + end + + describe 'existing dynamic behaviour' do + it 'keeps field readers' do + assert_equal 'Moby Dick', @entry.title.to_s + assert_respond_to @entry, :title + end + + it 'keeps convert_ and convert_!' do + entry = Entry.new(bibtex_type: :book, title: 'M\"{o}by') + + assert_equal 'Möby', entry.convert_latex.title.to_s + assert_equal 'M\"{o}by', entry.title.to_s + + entry.convert_latex! + + assert_equal 'Möby', entry.title.to_s + end + + it 'keeps from_' do + entry = Entry.new(bibtex_type: :book, title: 'M\"{o}by') + + assert_equal 'Möby', entry.from_latex.title.to_s + end + + it 'reports the ghost conversion methods via respond_to?' do + assert_respond_to @entry, :convert_latex + assert_respond_to @entry, :convert_latex! + assert_respond_to @entry, :from_latex + end + + it 'keeps delegating readers to a cross-referenced parent' do + bib = Bibliography.parse('@book{a, title = "A", editor = "E"} @incollection{a1, crossref = "a"}') + + assert_equal 'A', bib['a1'].title.to_s + assert_respond_to bib['a1'], :title + assert_respond_to bib['a1'], :editor + refute bib['a1'].field?(:title) + end + end + + describe 'real methods and operators' do + it 'keeps key= as the key writer, not a field' do + @entry.key = 'melville' + + assert_equal 'melville', @entry.key + refute @entry.field?(:key) + end + + it 'keeps type= as the type writer, not a field' do + @entry.type = :article + + assert_equal :article, @entry.type + refute @entry.field?(:type) + end + + it 'keeps []= as before' do + @entry[:isbn] = '978-0142437247' + + assert_equal '978-0142437247', @entry[:isbn].to_s + end + + it 'keeps the generated writers of required and name fields' do + @entry.title = 'Billy Budd' + @entry.author = 'Herman Melville' + + assert_equal 'Billy Budd', @entry.title.to_s + assert_equal 'Herman Melville', @entry.author.to_s + end + + it 'keeps == as before' do + other = Entry.new(bibtex_type: :book, bibtex_key: :moby, title: 'Moby Dick', keywords: 'whale') + + assert_equal other, @entry + refute_equal Entry.new(bibtex_type: :book, bibtex_key: :moby, title: 'Typee'), @entry + refute @entry.field?(:'=') + end + + it 'keeps <=> and <= as before' do + a = Entry.new(bibtex_type: :article, bibtex_key: :a) + b = Entry.new(bibtex_type: :book, bibtex_key: :b) + + assert_equal(-1, a <=> b) + assert_operator a, :<=, b + assert_operator b, :>, a + assert_empty a.fields + end + end + end + end +end diff --git a/test/bibtex/test_error_recovery.rb b/test/bibtex/test_error_recovery.rb new file mode 100644 index 0000000..42bbaf4 --- /dev/null +++ b/test/bibtex/test_error_recovery.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'helper' + +module BibTeX + # How the parser recovers from broken input: which objects survive, which + # become BibTeX::Error objects, and which inputs fail as a whole. + class ErrorRecoveryTest < Minitest::Spec + # errors.bib, split before each line that describes the next case. + SECTIONS = File.read(Test.fixtures(:errors), encoding: 'utf-8').split(/\n(?=[A-Z][^\n]*:\n)/).freeze + + UNBALANCED = %i[AT STRING LBRACE NAME EQ STRING_LITERAL E_UNBALANCED].freeze + UNTERMINATED = %i[AT STRING LBRACE NAME EQ STRING_LITERAL E_UNTERMINATED_STRING].freeze + + # The expected outcome for each section of errors.bib, in order. + EXPECTED = [ + [], + [['Error', UNBALANCED]], + [['Error', UNBALANCED]], + [['Error', UNTERMINATED]], + [['String', :foo]], + [['Error', %i[AT PREAMBLE LBRACE STRING_LITERAL E_UNTERMINATED_STRING]]], + [['Preamble']], + ParseError, + [['Error', %i[AT NAME LBRACE KEY NAME EQ STRING_LITERAL NAME NAME STRING_LITERAL E_UNBALANCED]]], + [['Entry', :book, 'key', %i[author title]]], + ParseError, + [['String', :bar]] + ].freeze + + OPTIONS = [{}, { strict: false }].freeze + + def summary(bib) + bib.map do |element| + name = element.class.name.split('::').last + case element + when Entry then [name, element.type, element.key, element.fields.keys] + when BibTeX::String then [name, element.key] + when BibTeX::Error then [name, element.trace.map(&:first)] + else [name] + end + end + end + + before { BibTeX.log.level = Logger::FATAL } + after { BibTeX.log.level = Logger::WARN } + + it 'splits errors.bib into one section per case' do + assert_equal EXPECTED.length, SECTIONS.length + end + + OPTIONS.each do |options| + describe "with #{options.inspect}" do + EXPECTED.each_with_index do |result, index| + it "parses section #{index} of errors.bib (#{SECTIONS[index].lines.first.strip})" do + if result == ParseError + assert_raises(ParseError) { BibTeX.parse(SECTIONS[index], options) } + else + assert_equal result, summary(BibTeX.parse(SECTIONS[index], options)) + end + end + end + + it 'fails on errors.bib as a whole' do + error = assert_raises(ParseError) { BibTeX.open(Test.fixtures(:errors), options) } + + assert_match(/\AFailed to parse BibTeX on value "\$end" \(\$end\) \[#, "@", "string", "\{", \[:foo, /, + error.message) + end + + it 'rejects an entry without a comma after the key or between fields' do + assert_raises(ParseError) { BibTeX.parse("@book{key\n\tauthor = \"An Author\"\n}", options) } + assert_raises(ParseError) { BibTeX.parse("@book{key,\n\tauthor = \"An Author\"\n\ttitle = \"The Title\"\n}", options) } + end + + it 'reports a braced value that is never closed as an unterminated-content error' do + assert_equal [['Error', %i[AT NAME LBRACE KEY NAME EQ LBRACE CONTENT E_UNTERMINATED_CONTENT]]], + summary(BibTeX.parse('@book{a, title = {x', options)) + end + end + end + + it 'rejects an entry without key unless allow_missing_keys is set' do + input = "@book{\n\tauthor = \"An Author\"\n}" + + assert_raises(ParseError) { BibTeX.parse(input) } + assert_raises(ParseError) { BibTeX.parse(input, strict: false) } + assert_equal [['Entry', :book, 'an-a', [:author]]], summary(BibTeX.parse(input, allow_missing_keys: true)) + end + + it 'finds an indented entry after free text in non-strict mode' do + assert_equal [['Entry', :book, 'a', [:title]]], + summary(BibTeX.parse("text\n @book{a, title = {x}}", strict: false)) + end + end +end diff --git a/test/bibtex/test_identifiers.rb b/test/bibtex/test_identifiers.rb new file mode 100644 index 0000000..b16e1f6 --- /dev/null +++ b/test/bibtex/test_identifiers.rb @@ -0,0 +1,384 @@ +# frozen_string_literal: true + +require 'helper' + +module BibTeX + # Identifier characters accepted by BibTeX::Lexer in + # field names, @string names, macro references and keys. + class IdentifiersTest < Minitest::Spec + # Characters BibTeX allows in identifiers, most of which the lexer used + # to reject. + EXAMPLES = %w[a/b x;y p[1] a|b a^b a~b a a@b].freeze + + MODES = { + 'strict' => {}, + 'non-strict' => { strict: false }, + 'strict with :strip_comments' => { strip_comments: true }, + 'non-strict with :strip_comments' => { strict: false, strip_comments: true } + }.freeze + + SLASH_KEY = 'DBLP:journals/dq/RossinK99' + + # Characters BibTeX excludes from identifiers that the lexer rejects today. + REJECTED = %w[' ( )].freeze + + # Keys accepted before the identifier characters were widened, mapped to the key they are parsed into. + EXISTING_KEYS = { + 'key:0' => 'key:0', 'a+b' => 'a+b', 'a;b' => 'a;b', %(a'b"c) => %(a'b"c), + 'fo{o' => 'fo{o', 'löwe' => 'löwe', ' spaced ' => 'spaced', 'a.b-c_d!e$f?g&h*i' => 'a.b-c_d!e$f?g&h*i' + }.freeze + + # Unterminated entries followed by the next object. + UNTERMINATED = [ + "@misc{a, title = {t},\n@book{b, title={t}}", + "@misc{a, title = {t}\n@book{b, title={t}}", + "@misc{a, title = t\n@book {b, title={t}}" + ].freeze + + # Long runs of '@a' that made lexing quadratic while a word starting + # with '@' could extend over later '@'s (about 16 s for n = 50_000). + RUN = 50_000 + ADVERSARIAL = { + "'@' + 'a@' * n + ' {'" => "@#{'a@' * RUN} {", + "an entry with '@a' * n + ' {'" => "@misc{k, #{'@a' * RUN} {", + "an entry ending in '@a' * n" => "@misc{k, #{'@a' * RUN}", + "a value of '@a' * n + ' x'" => "@misc{k, title = #{'@a' * RUN} x" + }.freeze + + def parse(input, options = {}) + BibTeX.parse(input, options) + end + + def tokens(input, options = {}) + Lexer.new(options).analyse(input).stack + end + + describe 'identifier characters' do + before { BibTeX.log.level = Logger::FATAL } + after { BibTeX.log.level = Logger::WARN } + + MODES.each do |mode, options| + describe "in #{mode} mode" do + EXAMPLES.each do |name| + it "lexes #{name} as a single field NAME token" do + assert_equal [[:AT, '@'], [:NAME, 'misc'], [:LBRACE, '{'], [:KEY, 'k'], + [:NAME, name], [:EQ, '='], [:LBRACE, '{'], [:CONTENT, 'v'], + [:RBRACE, '}'], [:RBRACE, '}'], [false, '$end']], + tokens("@misc{k, #{name} = {v}}", options) + end + + it "accepts #{name} as a field name" do + bib = parse("@misc{k, #{name} = {v}, title = {T}}", options) + + assert_empty bib.errors + assert_equal [name.to_sym, :title], bib['k'].fields.keys + assert_equal 'v', bib['k'][name].to_s + end + + it "accepts #{name} as a field name without surrounding spaces" do + bib = parse("@misc{k,#{name}={v},title={T}}", options) + + assert_empty bib.errors + assert_equal 'v', bib['k'][name].to_s + end + + it "accepts #{name} as an @string name" do + bib = parse("@string{ #{name} = {v} }", options) + + assert_empty bib.errors + assert_equal [name.to_sym], bib.strings.keys + assert_equal 'v', bib.strings[name.to_sym].value.to_s + end + + it "accepts #{name} as a macro reference" do + bib = parse("@string{ #{name} = {v} }\n@misc{k, title = #{name}}", options) + + assert_empty bib.errors + assert_equal name.to_sym, bib['k'][:title].to_sym + assert_equal 'v', bib.replace_strings['k'][:title].to_s + end + + it "accepts #{name} as a macro reference in a concatenation" do + bib = parse("@string{ #{name} = {v} }\n@misc{k, title = {a} # #{name} # \"b\"}", options) + + assert_empty bib.errors + assert_equal 'avb', bib.replace_strings.join_strings['k'][:title].to_s + end + + it "does not let #{name} swallow the next object" do + bib = parse("@misc{k, #{name} = {v}}\n@misc{k2, title = {T}}", options) + + assert_empty bib.errors + assert_equal %w[k k2], bib.entries.keys + end + + it "writes #{name} out so that it parses again" do + entry = parse("@misc{k, #{name} = {v}}", options)['k'] + reparsed = parse(entry.to_s, options)['k'] + + assert_equal entry.fields, reparsed.fields + end + end + + it 'keeps non-ASCII letters in field names, @string names and references' do + bib = parse("@string{ ä = {v} }\n@misc{k, ü = ä}", options) + + assert_empty bib.errors + assert_equal %i[ü], bib['k'].fields.keys + assert_equal :ä, bib['k'][:ü].to_sym + end + + it 'combines non-ASCII letters with the new characters' do + bib = parse("@string{ ä/b = {v} }\n@misc{k, ü<1> = ä/b}", options) + + assert_empty bib.errors + assert_equal %i[ü<1>], bib['k'].fields.keys + assert_equal :'ä/b', bib['k'][:'ü<1>'].to_sym + end + + it 'still rejects 1a as a field name' do + assert_raises(ParseError) { parse('@misc{k, 1a = {v}}', options) } + end + + it 'still rejects 1a as an @string name' do + assert_raises(ParseError) { parse('@string{ 1a = {v} }', options) } + end + + it 'still rejects 1a as a macro reference' do + assert_raises(ParseError) { parse('@misc{k, title = 1a}', options) } + end + + it 'still lexes a number that follows = as a NUMBER' do + assert_equal [:NUMBER, '1999'], tokens('@misc{k, year = 1999}', options)[-3] + end + + it 'still splits names at the characters BibTeX excludes' do + assert_equal [[:NAME, 'title'], [:EQ, '='], [:NAME, 'a'], [:SHARP, '#'], [:NAME, 'b'], [:COMMA, ','], + [:NAME, 'year'], [:EQ, '='], [:STRING_LITERAL, 'x'], [:RBRACE, '}']], + tokens('@misc{k,title=a#b,year="x"}', options)[4..-2] + end + + REJECTED.each do |char| + it "still does not accept #{char} inside a field name" do + bib = parse("@misc{k, a#{char}b = {v}}", options) + + assert_nil bib['k'] + refute_empty bib.errors + end + end + + it "accepts the key #{SLASH_KEY}" do + bib = parse("@article{#{SLASH_KEY}, title = {T}}", options) + + assert_empty bib.errors + assert_equal [SLASH_KEY], bib.entries.keys + assert_equal 'T', bib[SLASH_KEY].title.to_s + end + + it 'accepts keys with / and allow_missing_keys' do + bib = parse("@article{#{SLASH_KEY}, title = {T}}", options.merge(allow_missing_keys: true)) + + assert_empty bib.errors + assert_equal [SLASH_KEY], bib.entries.keys + end + + it 'still accepts a missing key with allow_missing_keys' do + bib = parse('@article{, title = {T}}', options.merge(allow_missing_keys: true)) + + assert_empty bib.errors + assert_equal 'T', bib.data[0].title.to_s + end + + EXISTING_KEYS.each do |source, key| + it "still accepts the key #{source.inspect}" do + bib = parse("@misc{#{source}, title = {T}}", options) + + assert_empty bib.errors + assert_equal [key], bib.entries.keys + end + end + end + end + + describe '% in names' do + it 'is part of a field name, @string name and reference without :strip_comments' do + bib = parse("@string{ a/b%c = {v} }\n@misc{k, x%y = a/b%c}") + + assert_empty bib.errors + assert_equal [:'x%y'], bib['k'].fields.keys + assert_equal :'a/b%c', bib['k'][:'x%y'].to_sym + end + + it 'is part of a field name in non-strict mode without :strip_comments' do + bib = parse('@misc{k, a/b%c = {v}}', strict: false) + + assert_equal [:'a/b%c'], bib['k'].fields.keys + end + + it 'starts a comment after a new-style name with :strip_comments' do + bib = parse("@string{ a/b = {v} }\n@misc{k, x;y% comment\n = a/b% comment\n}", strip_comments: true) + + assert_empty bib.errors + assert_equal [:'x;y'], bib['k'].fields.keys + assert_equal :'a/b', bib['k'][:'x;y'].to_sym + end + + it 'starts a comment after a key with / and :strip_comments' do + bib = parse("@article{% before\n#{SLASH_KEY}, % after\n title = {T} % end\n}", strip_comments: true) + + assert_empty bib.errors + assert_equal [SLASH_KEY], bib.entries.keys + end + + it 'stays inside the key without :strip_comments' do + bib = parse('@misc{a%b, title = {T}}') + + assert_equal ['a%b'], bib.entries.keys + end + end + + describe '@ inside a name' do + it 'does not start a new object in strict mode' do + bib = parse("@misc{k, a@b = {v}, title = {T}}\n@misc{k2, title = {T2}}") + + assert_equal %w[k k2], bib.entries.keys + assert_equal %i[a@b title], bib['k'].fields.keys + end + + it 'does not start a new object in non-strict mode' do + bib = parse("@misc{k,\n a@b = {v},\n title = c@d\n}\n@misc{k2, title = {T2}}", strict: false) + + assert_equal %w[k k2], bib.entries.keys + assert_equal :'c@d', bib['k'][:title].to_sym + end + + it 'still starts a new object at @ between objects in strict mode' do + bib = parse('@misc{k, title = {T}} junk @misc{k2, title = {T2}}') + + assert_equal %w[k k2], bib.entries.keys + end + end + + describe '@ at the start of a name' do + MODES.each do |mode, options| + describe "in #{mode} mode" do + it 'lexes @foo before = as a field NAME' do + assert_equal [[:AT, '@'], [:NAME, 'misc'], [:LBRACE, '{'], [:KEY, 'k'], + [:NAME, '@foo'], [:EQ, '='], [:LBRACE, '{'], [:CONTENT, 'x'], + [:RBRACE, '}'], [:RBRACE, '}'], [false, '$end']], + tokens('@misc{k, @foo = {x}}', options) + end + + it 'accepts @foo as a field name' do + bib = parse("@misc{k, @foo = {x}, title = {T}}\n@misc{k2, title = {T2}}", options) + + assert_empty bib.errors + assert_equal %w[k k2], bib.entries.keys + assert_equal %i[@foo title], bib['k'].fields.keys + end + + it 'accepts @m as an @string name and as a macro reference before , } and #' do + bib = parse("@string{@m = {x}}\n@misc{k, a = @m, b = @m # {y}, c = {y} # @m}", options) + + assert_empty bib.errors + assert_equal %i[@m], bib.strings.keys + assert_equal :@m, bib['k'][:a].to_sym + resolved = bib.replace_strings.join_strings['k'] + + assert_equal 'x', resolved[:a].to_s + assert_equal 'xy', resolved[:b].to_s + assert_equal 'yx', resolved[:c].to_s + end + + it 'still splits two entries separated by a trailing comma' do + bib = parse("@misc{a, title={t}},\n@book{b, title={t}}", options) + + assert_empty bib.errors + assert_equal %w[a b], bib.entries.keys + end + + it 'still accepts a trailing comma inside an entry before the next one' do + bib = parse("@misc{a, title={t},\n}\n@book{b, title={t}}", options) + + assert_empty bib.errors + assert_equal %w[a b], bib.entries.keys + end + + ADVERSARIAL.each do |label, input| + it "lexes #{label} in linear time" do + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + Lexer.new(options).analyse(input) + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + + # About 0.1 s when linear; generous for slow CI machines. + assert_operator elapsed, :<, 5, "took #{elapsed.round(2)} s" + end + end + + it "ends a name that starts with '@' before the next '@'" do + assert_equal [[:NAME, '@a'], [:EQ, '=']], tokens('@misc{k, @a = {x}}', options)[4, 2] + refute_includes tokens('@misc{k, @a@b = {x}}', options), [:NAME, '@a@b'] + assert_equal [:NAME, 'a@b@c'], tokens('@misc{k, a@b@c = {x}}', options)[4] + end + + UNTERMINATED.each do |input| + it "still starts a new object after an unterminated entry: #{input.inspect}" do + stack = tokens(input, options) + start = stack.rindex([:AT, '@']) + + assert_equal [[:AT, '@'], [:NAME, 'book'], [:LBRACE, '{'], [:KEY, 'b']], stack[start, 4] + refute(stack.any? { |type, value| type == :NAME && value.start_with?('@') }) + end + end + end + end + + it 'skips TeX comments between @foo and = with :strip_comments' do + bib = parse("@misc{k, @foo % comment\n = {x}}", strip_comments: true) + + assert_empty bib.errors + assert_equal %i[@foo], bib['k'].fields.keys + end + + it 'allows @ at the start of the last word of a non-strict multi-word name' do + bib = parse('@misc{k, early @date = {v}}', strict: false) + + assert_equal %i[early_@date], bib['k'].fields.keys + end + end + + describe 'non-strict field names made of several words' do + it 'uses the new characters in every word' do + bib = parse('@misc{k, a/b x;y p[1] = {v}, title = {T}}', strict: false) + + assert_empty bib.errors + assert_equal %i[a/b_x;y_p[1] title], bib['k'].fields.keys + end + + EXAMPLES.each do |name| + it "joins words containing #{name}" do + bib = parse("@misc{k, early #{name}\tdate = {v}}", strict: false) + + assert_equal [:"early_#{name}_date"], bib['k'].fields.keys + end + end + + it 'combines with :strip_comments' do + bib = parse("@misc{k, % a c = {commented out}\n a|b c~d = {v} % comment\n}", strict: false, strip_comments: true) + + assert_equal [:'a|b_c~d'], bib['k'].fields.keys + end + + it 'keeps % in the words without :strip_comments' do + bib = parse('@misc{k, a%b c^d = {v}}', strict: false) + + assert_equal [:'a%b_c^d'], bib['k'].fields.keys + end + + it 'still raises for several words in strict mode' do + assert_raises(ParseError) { parse('@misc{k, a/b x;y = {v}}') } + end + end + end + end +end diff --git a/test/bibtex/test_parser.rb b/test/bibtex/test_parser.rb index 9cfc3ac..e010ba1 100644 --- a/test/bibtex/test_parser.rb +++ b/test/bibtex/test_parser.rb @@ -268,6 +268,10 @@ class ParserTest < Minitest::Spec assert_equal 3, @bib.meta_contents.length end + it 'keeps comments and meta content in document order' do + assert_equal [MetaContent, Comment, MetaContent, Comment, MetaContent], @bib.data.map(&:class) + end + it 'should parse @comment content as string' do assert_equal ' A comment can contain pretty much anything ', @bib.comments[0].content assert_equal %(\n@string{ foo = "bar" }\n\n@string{ bar = "foo" }\n), @bib.comments[1].content @@ -367,5 +371,39 @@ class ParserTest < Minitest::Spec assert_equal '1=highlight', bib[:foo][:'author+an'] end end + + describe 'given meta content between objects' do + it 'drops the line breaks that only separate objects' do + input = "intro\n@book{a, title = {x}}\n\n@book{b, title = {y}}\n@comment{c}\ntail\n" + bib = Parser.new(include: [:meta_content]).parse(input) + + assert_equal ["intro\n", '', '', "\ntail\n"], bib.meta_contents.map(&:to_s) + end + end + + describe 'given TeX comments and :strip_comments' do + it 'skips a comment that follows white space before the key' do + bib = Parser.new(strip_comments: true).parse("@misc{ % c\n foo, a = {x}}") + + assert_equal %w[foo], bib.entries.keys + assert_equal %i[a], bib['foo'].fields.keys + end + end + + describe 'given malformed entry types and keys' do + it 'rejects entry types with characters BibTeX does not allow there' do + assert_raises(ParseError) { Parser.new.parse('@bo/ok{a, title = {x}}') } + assert_raises(ParseError) { Parser.new.parse('@bo[k]{a, title = {x}}') } + end + + it 'rejects a line break or tab between the key and its comma' do + assert_raises(ParseError) { Parser.new.parse("@book{a\n, title = {x}}") } + assert_raises(ParseError) { Parser.new.parse("@book{a\t, title = {x}}") } + end + + it 'keeps a space between the key and its comma as part of a valid entry' do + assert_equal %w[a], Parser.new.parse('@book{a , title = {x}}').entries.keys + end + end end end diff --git a/test/test_bibtex.rb b/test/test_bibtex.rb index 1c0ed9f..4d204ad 100644 --- a/test/test_bibtex.rb +++ b/test/test_bibtex.rb @@ -19,6 +19,21 @@ class TestBibtex < Minitest::Test } BIBTEX + # The entry in test/fixtures/roundtrip.bib. + RAILS_ATTRIBUTES = { + bibtex_type: :book, + bibtex_key: 'rails', + address: 'Raleigh, North Carolina', + author: 'Ruby, Sam and Thomas, Dave and Hansson Heinemeier, David', + booktitle: 'Agile Web Development with Rails', + edition: 'third', + keywords: 'ruby, rails', + publisher: 'The Pragmatic Bookshelf', + series: 'The Facets of Ruby', + title: 'Agile Web Development with Rails', + year: '2009' + }.freeze + def setup; end def teardown; end @@ -37,6 +52,10 @@ def test_no_bibtex refute_nil(bib) assert_instance_of(BibTeX::Bibliography, bib) assert_empty(bib) + + with_meta = BibTeX::Bibliography.open(Test.fixtures(:no_bibtex), include: [:meta_content]) + + assert_equal([BibTeX::MetaContent], with_meta.data.map(&:class)) end def test_decoret @@ -45,7 +64,16 @@ def test_decoret assert_equal(15, bib.length) assert_equal([BibTeX::Entry, BibTeX::Comment, BibTeX::String, BibTeX::Preamble], bib.data.map(&:class).uniq) assert_equal('py03', bib.data[0].key) - # TODO: missing assertions + end + + def test_decoret_objects + bib = BibTeX::Bibliography.open(Test.fixtures(:decoret), debug: false) + + assert_equal(%w[py03 key03 key01 key02 steward03 sweig42], bib.entries.keys) + assert_equal(%i[mar firstname lastname email maintainer], bib.strings.keys) + assert_equal({ BibTeX::Entry => 6, BibTeX::Comment => 2, BibTeX::String => 5, BibTeX::Preamble => 2 }, + bib.data.map(&:class).tally) + assert_empty(bib.errors) end def test_decoret_py03 @@ -64,42 +92,30 @@ def test_decoret_key03 assert_equal('A {bunch {of} braces {in}} title', entry[:title]) end - # def test_errors - # bib = BibTeX.open(Test.fixtures(:errors), :debug => false) - # #refute_nil(bib) - # end - def test_bibdesk bib = BibTeX::Bibliography.open(Test.fixtures(:bibdesk), debug: false) assert_equal 3, bib.length - assert_equal 'rails', bib[0].key + assert_equal %w[rails dragon pickaxe], bib.entries.keys + assert_empty bib.errors + assert_equal %i[address author booktitle date-added date-modified edition isbn keywords publisher series title year], + bib['rails'].fields.keys assert_equal '2010-08-05 10:06:32 +0200', bib[:dragon]['date-modified'] end def test_roundtrip - # file = File.read(Test.fixtures(:roundtrip)) - # bib = BibTeX.parse(file, :debug => false) - # assert_equal file.gsub(/[\s]+/, ''), bib.to_s.gsub(/[\s]+/, '') + file = File.read(Test.fixtures(:roundtrip)) + bib = BibTeX.parse(file) + + assert_equal file.gsub(/\s+/, ''), bib.to_s.gsub(/\s+/, '') end def test_construct - # file = File.read(Test.fixtures(:roundtrip)) - # bib = BibTeX::Bibliography.new - # bib << BibTeX::Entry.new({ - # :bibtex_type => :book, - # :key => 'rails', - # :address => 'Raleigh, North Carolina', - # :author => 'Ruby, Sam and Thomas, Dave and Hansson Heinemeier, David', - # :booktitle => 'Agile Web Development with Rails', - # :edition => 'third', - # :keywords => 'ruby, rails', - # :publisher => 'The Pragmatic Bookshelf', - # :series => 'The Facets of Ruby', - # :title => 'Agile Web Development with Rails', - # :year => '2009' - # }) - # assert_equal(file.gsub(/[\s]+/, ''), bib.to_s.gsub(/[\s]+/, '')) + file = File.read(Test.fixtures(:roundtrip)) + bib = BibTeX::Bibliography.new + bib << BibTeX::Entry.new(RAILS_ATTRIBUTES.dup) # Entry.new deletes keys from its argument + + assert_equal file.gsub(/\s+/, ''), bib.to_s.gsub(/\s+/, '') end def test_parse