Commit 53f019c7 authored by valenting's avatar valenting Committed by Pier Angelo Vendrame
Browse files

Bug 2025888 - Make IncrementalTokenizer read entire buffer r=necko-reviewers,jesup 

Differential Revision: https://phabricator.services.mozilla.com/D290085
parent 7dac05b1

xpcom/ds/IncrementalTokenizer.cpp

+5 −15
Original line number Diff line number Diff line
@@ -18,15 +18,7 @@ IncrementalTokenizer::IncrementalTokenizer(Consumer&& aConsumer, 
                                           const char* aWhitespaces,

                                           const char* aAdditionalWordChars,

                                           uint32_t aRawMinBuffered)

    : TokenizerBase(aWhitespaces, aAdditionalWordChars)

#ifdef DEBUG

      ,

      mConsuming(false)

#endif

      ,

      mNeedMoreInput(false),

      mRollback(false),

      mInputCursor(0),

    : TokenizerBase(aWhitespaces, aAdditionalWordChars),

      mConsumer(std::move(aConsumer)) {

  mInputFinished = false;

  mMinRawDelivery = aRawMinBuffered;
@@ -60,9 +52,6 @@ nsresult IncrementalTokenizer::FeedInput(nsIInputStream* aInput, 
        std::min<nsCString::index_type>(aCount, PR_UINT32_MAX - remainder);



    if (!load) {

      // To keep the API simple, we fail if the input data buffer if filled.

      // It's highly unlikely there will ever be such amout of data cumulated

      // unless a logic fault in the consumer code.

      NS_ERROR("IncrementalTokenizer consumer not reading data?");

      return NS_ERROR_OUT_OF_MEMORY;

    }
@@ -76,13 +65,14 @@ nsresult IncrementalTokenizer::FeedInput(nsIInputStream* aInput, 
    uint32_t read;

    rv = aInput->Read(buffer, load, &read);

    if (NS_SUCCEEDED(rv)) {

      // remainder + load fits the uint32_t size, so must remainder + read.

      mInput.SetLength(remainder + read);

      aCount -= read;

    }

  }



  if (NS_SUCCEEDED(rv)) {

    rv = Process();

  }

  }



  return rv;

}

xpcom/ds/IncrementalTokenizer.h

+4 −4
Original line number Diff line number Diff line
@@ -101,21 +101,21 @@ class IncrementalTokenizer : public TokenizerBase<char> { 


#ifdef DEBUG

  // True when inside the consumer callback, used only for assertions.

  bool mConsuming;

  bool mConsuming{false};

#endif  // DEBUG

  // Modifyable only from the Consumer callback, tells the parser to break,

  // rollback and wait for more input.

  bool mNeedMoreInput;

  bool mNeedMoreInput{false};

  // Modifyable only from the Consumer callback, tells the parser to rollback

  // and parse the input again, with (if modified) new settings of the

  // tokenizer.

  bool mRollback;

  bool mRollback{false};

  // The input buffer.  Updated with each call to Feed/FinishInput.

  nsCString mInput;

  // Numerical index pointing at the current cursor position.  We don't keep

  // direct reference to the string buffer since the buffer gets often

  // reallocated.

  nsCString::index_type mInputCursor;

  nsCString::index_type mInputCursor{0};

  // Refernce to the consumer function.

  Consumer mConsumer;

};