Ongoing spam-filtering work for the front end (not yet exposed to the user in... (7fbbf990) · Commits · The Tor Project / Applications / Mullvad Browser

mailnews/base/resources/content/mailCommands.js

+19 −142

Original line number	Diff line number	Diff line
		@@ -444,184 +444,61 @@ function getJunkmailComponent()
		}
		}

		function analyze(aMessage, aNextFunction)
		function analyze(aMsgHdr, aNextFunction)
		{
		var listener = {
		onMessageClassified: function(aMsgURL, aClassification)
		{
		dump(aMsgURL + ' is ' + (aClassification == nsIJunkMailPlugin.JUNK ? 'JUNK' : 'GOOD') + '\n');
		dump(aMsgURL + ' is '
		+ (aClassification == nsIJunkMailPlugin.JUNK
		? 'JUNK' : 'GOOD') + '\n');

		// XXX TODO, make the cut off 50, like in nsMsgSearchTerm.cpp
		var score = (aClassification == nsIJunkMailPlugin.JUNK ? "100" : "0");
		aMessage.setStringProperty("junkscore", score);
		var score =
		aClassification == nsIJunkMailPlugin.JUNK ? "100" : "0";

		// set these props via the db (instead of the message header
		// directly) so that the nsMsgDBView knows to update the UI
		//
		var db = aMsgHdr.folder.getMsgDatabase(msgWindow);
		db.setStringProperty(aMsgHdr.messageKey, "junkscore", score);
		db.setStringProperty(aMsgHdr.messageKey, "junkscoreorigin",
		"plugin");
		aNextFunction();
		}
		};

		// XXX TODO jumping through hoops here.
		var messageURI = aMessage.folder.generateMessageURI(aMessage.messageKey) + "?fetchCompleteMessage=true";
		var messageURI = aMsgHdr.folder.generateMessageURI(aMsgHdr.messageKey)
		+ "?fetchCompleteMessage=true";
		gJunkmailComponent.classifyMessage(messageURI, listener);
		}

		function analyzeFolder()
		{
		function processNext()
		{
		if (messages.hasMoreElements()) {
		// XXX TODO jumping through hoops here.
		var message = messages.getNext().QueryInterface(nsIMsgDBHdr);
		while (!message.isRead) {
		if (!messages.hasMoreElements()) {
		gJunkmailComponent.batchUpdate = false;
		return;
		}
		message = messages.getNext().QueryInterface(nsIMsgDBHdr);
		}
		analyze(message, processNext);
		}
		else {
		gJunkmailComponent.batchUpdate = false;
		}
		}

		getJunkmailComponent();
		var folder = GetFirstSelectedMsgFolder();
		var messages = folder.getMessages(msgWindow);
		gJunkmailComponent.batchUpdate = true;
		processNext();
		}

		function analyzeMessages()
		{
		function processNext()
		{
		if (counter < messages.length) {
		// XXX TODO jumping through hoops here.
		var messageUri = messages[counter];
		var message = messenger.messageServiceFromURI(messageUri).messageURIToMsgHdr(messageUri);

		++counter;
		while (!message.isRead) {
		if (counter == messages.length) {
		dump('[bayesian filter message analysis complete.]\n');
		gJunkmailComponent.mBatchUpdate = false;
		return;
		}
		messageUri = messages[counter];
		message = messenger.messageServiceFromURI(messageUri).messageURIToMsgHdr(messageUri);
		++counter;
		}
		analyze(message, processNext);
		}
		else {
		dump('[bayesian filter message analysis complete.]\n');
		gJunkmailComponent.batchUpdate = false;
		gJunkmailComponent.endBatch();
		}
		}

		getJunkmailComponent();
		var messages = GetSelectedMessages();
		var counter = 0;
		gJunkmailComponent.batchUpdate = true;
		gJunkmailComponent.startBatch();
		dump('[bayesian filter message analysis begins.]\n');
		processNext();
		}

		function writeHash()
		{
		getJunkmailComponent();
		gJunkmailComponent.mTable.writeHash();
		}

		function mark(aMessage, aSpam, aNextFunction)
		{
		// XXX TODO jumping through hoops here.
		var score = aMessage.getStringProperty("junkscore");

		var oldClassification = ((score == "100") ? nsIJunkMailPlugin.JUNK :
		(score == "0") ? nsIJunkMailPlugin.GOOD : nsIJunkMailPlugin.UNCLASSIFIED);

		var newClassification = (aSpam ? nsIJunkMailPlugin.JUNK : nsIJunkMailPlugin.GOOD);

		var messageURI = aMessage.folder.generateMessageURI(aMessage.messageKey) + "?fetchCompleteMessage=true";

		var listener = (aNextFunction == null ? null :
		{
		onMessageClassified: function(aMsgURL, aClassification)
		{
		aNextFunction();
		}
		});

		gJunkmailComponent.setMessageClassification(messageURI, oldClassification, newClassification, listener);
		}

		function JunkSelectedMessages(setAsJunk)
		{
		getJunkmailComponent();
		var messages = GetSelectedMessages();

		// start the batch of messages
		//
		gJunkmailComponent.batchUpdate = true;

		// mark each one
		//
		for ( var msg in messages ) {
		var message = messenger.messageServiceFromURI(messages[msg])
		.messageURIToMsgHdr(messages[msg]);
		mark(message, setAsJunk, null);
		}

		// end the batch (tell the component to write out its data)
		//
		gJunkmailComponent.batchUpdate = false;

		// this actually sets the score on the selected messages
		// so we need to call it after we call mark()
		gDBView.doCommand(setAsJunk ? nsMsgViewCommandType.junk
		: nsMsgViewCommandType.unjunk);
		}

		// temporary
		function markFolderAsJunk(aSpam)
		{
		getJunkmailComponent();

		var listener = {
		messageCount: 0,
		onMessageClassified: function(aMsgURL, aClassification)
		{
		if (--this.messageCount == 0) {
		dump('[folder marking complete.]\n');
		gJunkmailComponent.batchUpdate = false;
		}
		}
		};

		var process = function(aMessage)
		{
		var score = aMessage.getStringProperty("junkscore");
		var oldClassification = ((score == "100") ? nsIJunkMailPlugin.JUNK :
		(score == "0") ? nsIJunkMailPlugin.GOOD : nsIJunkMailPlugin.UNCLASSIFIED);
		var newClassification = (aSpam ? nsIJunkMailPlugin.JUNK : nsIJunkMailPlugin.GOOD);
		var messageURI = aMessage.folder.generateMessageURI(aMessage.messageKey) + "?fetchCompleteMessage=true";

		++listener.messageCount;
		gJunkmailComponent.setMessageClassification(messageURI, oldClassification, newClassification, listener);
		}

		var folder = GetFirstSelectedMsgFolder();
		var messages = folder.getMessages(msgWindow);
		var newScore = (aSpam ? "100" : "0");

		gJunkmailComponent.batchUpdate = true;
		dump('[folder marking starting.]\n');

		while (messages.hasMoreElements()) {
		var message = messages.getNext().QueryInterface(nsIMsgDBHdr);
		process(message);
		// now set the score
		// XXX TODO invalidate the row
		message.setStringProperty("junkscore", newScore);
		}
		}

mailnews/base/resources/content/mailWindowOverlay.xul

+2 −4

Original line number	Diff line number	Diff line
		@@ -1435,11 +1435,9 @@ Rights Reserved.
		accesskey="&junkMailCmd.accesskey;"
		oncommand="MsgJunkMail()"/>
		<menuseparator/>
		<menuitem label="Mark All Messages In Folder As Junk" oncommand="markFolderAsJunk(true);"/>
		<menuitem label="Mark All Messages In Folder As Not Junk" oncommand="markFolderAsJunk(false);"/>
		<menuitem label="Analyze Folder" oncommand="analyzeFolder();"/>
		<menuitem label="Mark Selected Messages As Junk" oncommand="JunkSelectedMessages(true);"/>
		<menuitem label="Mark Selected Messages As Not Junk" oncommand="JunkSelectedMessages(false);"/>
		<menuitem label="Analyze Messages" oncommand="analyzeMessages();"/>
		<menuitem label="Write DB" oncommand="writeHash();"/>
		<menuseparator/>
		-->
		<menuitem label="&filtersApply.label;"

mailnews/base/search/public/nsIMsgFilterPlugin.idl

+14 −7

Original line number	Diff line number	Diff line
		@@ -63,10 +63,11 @@ interface nsIMsgFilterPlugin : nsISupports
		/**
		* When a large number of messages are going to be trained or filtered,
		* it's typically not worth it to write everything to disk after
		* each message. The calling code will set batchUpdate to true before
		* starting a set messages, and then to false once it has finished.
		* each message. The calling code will call startBatch() before
		* starting a set messages, and then endBatch() once it has finished.
		*/
		attribute boolean batchUpdate;
		void startBatch();
		void endBatch();
		};

		/**
		@@ -104,9 +105,15 @@ interface nsIJunkMailPlugin : nsIMsgFilterPlugin
		/**
		* Called when a user forces the classification of a message. Should
		* cause the training set to be updated appropriately.
		*
		* @arg aMsgURI URI of the message to be classified
		* @arg aOldUserClassification Was it previous manually classified
		* by the user? If so, how?
		* @arg aNewClassification New manual classification.
		* @arg aListener Callback
		*/
		void setMessageClassification(in string aMsgURI,
		in nsMsgJunkStatus aOldClassification,
		void setMessageClassification(
		in string aMsgURI, in nsMsgJunkStatus aOldUserClassification,
		in nsMsgJunkStatus aNewClassification,
		in nsIJunkMailClassificationListener aListener);
		};

mailnews/base/search/src/nsMsgSearchTerm.cpp

+8 −4

Original line number	Diff line number	Diff line
		@@ -1243,10 +1243,14 @@ nsresult nsMsgSearchTerm::MatchJunkStatus(const char aJunkScore, PRBool pResul

		nsMsgJunkStatus junkStatus;
		if (aJunkScore && *aJunkScore) {
		// I set the cut off at 50. this may change
		// cut off set at 50. this may change
		// it works for our bayesian plugin, as "0" is good, and "100" is junk
		// but it might need tweaking for other plugins
		junkStatus = (atoi(aJunkScore) > 50) ? nsIJunkMailPlugin::JUNK : nsIJunkMailPlugin::GOOD;
		if ( atoi(aJunkScore) > 50 ) {
		junkStatus = nsIJunkMailPlugin::JUNK;
		} else {
		junkStatus = nsIJunkMailPlugin::GOOD;
		}
		}
		else
		junkStatus = nsIJunkMailPlugin::UNCLASSIFIED;

mailnews/base/src/nsMsgDBView.cpp

+166 −6

Original line number	Diff line number	Diff line
		@@ -115,6 +115,7 @@ NS_INTERFACE_MAP_BEGIN(nsMsgDBView)
		NS_INTERFACE_MAP_ENTRY(nsIDBChangeListener)
		NS_INTERFACE_MAP_ENTRY(nsITreeView)
		NS_INTERFACE_MAP_ENTRY(nsIObserver)
		NS_INTERFACE_MAP_ENTRY(nsIJunkMailClassificationListener)
		NS_INTERFACE_MAP_END

		nsMsgDBView::nsMsgDBView()
		@@ -133,6 +134,7 @@ nsMsgDBView::nsMsgDBView()
		mIsNews = PR_FALSE;
		mDeleteModel = nsMsgImapDeleteModels::MoveToTrash;
		m_deletingRows = PR_FALSE;
		mOutstandingJunkBatches = 0;

		/* mCommandsNeedDisablingBecauseOffline - A boolean that tell us if we needed to disable commands because we're offline w/o a downloaded msg select */

		@@ -680,7 +682,8 @@ nsresult nsMsgDBView::FetchLabel(nsIMsgHdr aHdr, PRUnichar * aLabelString)
		NS_ENSURE_ARG_POINTER(aHdr);
		NS_ENSURE_ARG_POINTER(aLabelString);

		aHdr->GetLabel(&label);
		rv = aHdr->GetLabel(&label);
		NS_ENSURE_SUCCESS(rv, rv);

		// we don't care if label is not between 1 and PREF_LABELS_MAX inclusive.
		if ((label < 1) \|\| (label > PREF_LABELS_MAX))
		@@ -2067,6 +2070,15 @@ nsMsgDBView::ApplyCommandToIndices(nsMsgViewCommandTypeValue command, nsMsgViewI
		nsresult rv = NS_OK;
		nsMsgKeyArray imapUids;

		// if numIndices == 0, return quietly, just in case
		//
		if (numIndices == 0) {
		return NS_OK;
		}

		NS_ASSERTION(numIndices >= 0, "nsMsgDBView::ApplyCommandToIndices(): "
		"numIndices is negative!");

		nsCOMPtr <nsIMsgImapMailFolder> imapFolder = do_QueryInterface(m_folder);
		PRBool thisIsImapFolder = (imapFolder != nsnull);

		@@ -2076,6 +2088,39 @@ nsMsgDBView::ApplyCommandToIndices(nsMsgViewCommandTypeValue command, nsMsgViewI
		rv = DeleteMessages(mMsgWindow, indices, numIndices, PR_TRUE);
		else
		{
		nsCOMPtr<nsIJunkMailPlugin> junkPlugin;

		// if this is a junk command, start a batch. The batch will be ended
		// in the last callback.
		//
		if ( command == nsMsgViewCommandType::junk
		\|\| command == nsMsgViewCommandType::unjunk ) {

		// get the folder from the first item (if it's the search view,
		// only one item can be touched at a time; if a regular folder view,
		// all items will have the same folder).
		//
		nsCOMPtr<nsIMsgFolder> folder;
		rv = GetFolderForViewIndex(GetAt(indices[0]), getter_AddRefs(folder));
		NS_ENSURE_SUCCESS(rv, rv);

		nsCOMPtr<nsIMsgIncomingServer> server;
		rv = folder->GetServer(getter_AddRefs(server));
		NS_ENSURE_SUCCESS(rv, rv);

		nsCOMPtr<nsIMsgFilterPlugin> filterPlugin;
		rv = server->GetSpamFilterPlugin(getter_AddRefs(filterPlugin));
		NS_ENSURE_SUCCESS(rv, rv);

		junkPlugin = do_QueryInterface(filterPlugin, &rv);
		NS_ENSURE_SUCCESS(rv, rv);

		rv = junkPlugin->StartBatch();
		NS_ENSURE_SUCCESS(rv, rv);

		mOutstandingJunkBatches++;
		}

		for (int32 i = 0; i < numIndices; i++)
		{
		if (thisIsImapFolder && command != nsMsgViewCommandType::markThreadRead)
		@@ -2110,10 +2155,12 @@ nsMsgDBView::ApplyCommandToIndices(nsMsgViewCommandTypeValue command, nsMsgViewI
		rv = SetLabelByIndex(indices[i], (command - nsMsgViewCommandType::label0));
		break;
		case nsMsgViewCommandType::junk:
		rv = SetStringPropertyByIndex(indices[i], "junkscore", "100");
		rv = SetJunkScoreByIndex(junkPlugin.get(), indices[i],
		nsIJunkMailPlugin::JUNK, (i == numIndices-1));
		break;
		case nsMsgViewCommandType::unjunk:
		rv = SetStringPropertyByIndex(indices[i], "junkscore", "0");
		rv = SetJunkScoreByIndex(junkPlugin.get(), indices[i],
		nsIJunkMailPlugin::GOOD, (i == numIndices-1));
		break;
		case nsMsgViewCommandType::undeleteMsg:
		break; // this is completely handled in the imap code below.
		@@ -2388,6 +2435,120 @@ nsresult nsMsgDBView::SetStringPropertyByIndex(nsMsgViewIndex index, const char
		return rv;
		}

		nsresult nsMsgDBView::SetJunkScoreByIndex(nsIJunkMailPlugin *aJunkPlugin,
		nsMsgViewIndex aIndex,
		nsMsgJunkStatus aNewClassification,
		PRBool aIsLastInBatch)
		{

		// get the message header (need this to get string properties)
		//
		nsCOMPtr <nsIMsgDBHdr> msgHdr;
		nsresult rv = GetMsgHdrForViewIndex(aIndex, getter_AddRefs(msgHdr));
		NS_ENSURE_SUCCESS(rv, rv);

		// get the old junk score
		//
		nsXPIDLCString junkScoreStr;
		rv = msgHdr->GetStringProperty("junkscore", getter_Copies(junkScoreStr));

		// and the old origin
		//
		nsXPIDLCString oldOriginStr;
		rv = msgHdr->GetStringProperty("junkscoreorigin",
		getter_Copies(oldOriginStr));

		// if this was not classified by the user, say so
		//
		nsMsgJunkStatus oldUserClassification;
		if (oldOriginStr.get()[0] != 'u') {
		oldUserClassification = nsIJunkMailPlugin::UNCLASSIFIED;
		} else {
		// otherwise, pass the actual user classification
		//
		if (junkScoreStr.IsEmpty()) {
		oldUserClassification = nsIJunkMailPlugin::UNCLASSIFIED;
		} else if (atoi(junkScoreStr) > 50) {
		oldUserClassification = nsIJunkMailPlugin::JUNK;
		} else {
		oldUserClassification = nsIJunkMailPlugin::GOOD;
		}
		}

		// get the URI for this message so we can pass it to the plugin
		//
		nsXPIDLCString uri;
		rv = GetURIForViewIndex(aIndex, getter_Copies(uri));
		NS_ENSURE_SUCCESS(rv, rv);

		if ( aIsLastInBatch ) {
		// if there's already a batch in progress, just replace the URI,
		// thus causing the batches to be coalesced
		//
		mLastJunkUriInBatch = uri;
		}

		// tell the plugin about this change, so that it can (potentially)
		// adjust its database appropriately
		//
		rv = aJunkPlugin->SetMessageClassification(
		uri, oldUserClassification, aNewClassification, this);
		NS_ENSURE_SUCCESS(rv, rv);

		// set the junk score on the message itself
		//
		rv = SetStringPropertyByIndex(
		aIndex, "junkscore",
		aNewClassification == nsIJunkMailPlugin::JUNK ? "100" : "0");
		NS_ENSURE_SUCCESS(rv, rv);

		// this routine is only reached if the user someone touched the UI
		// and told us the junk status of this message.
		//
		rv = SetStringPropertyByIndex(aIndex, "junkscoreorigin", "user");
		NS_ENSURE_SUCCESS(rv, rv);

		return NS_OK;
		}

		NS_IMETHODIMP
		nsMsgDBView::OnMessageClassified(const char *aMsgUrl,
		nsMsgJunkStatus aClassification)
		{
		// is this the last url in the batch?
		//
		if ( mLastJunkUriInBatch.Equals(aMsgUrl) ) {

		// XXX are we allowed to assume m_folder exists here?
		//
		nsCOMPtr<nsIMsgIncomingServer> server;
		nsresult rv = m_folder->GetServer(getter_AddRefs(server));
		NS_ENSURE_SUCCESS(rv, rv);

		// get the filter, and QI to the interface we want
		//
		nsCOMPtr<nsIMsgFilterPlugin> filterPlugin;
		rv = server->GetSpamFilterPlugin(getter_AddRefs(filterPlugin));
		NS_ENSURE_SUCCESS(rv, rv);

		nsCOMPtr<nsIJunkMailPlugin> junkPlugin =
		do_QueryInterface(filterPlugin, &rv);
		NS_ENSURE_SUCCESS(rv, rv);

		// close out existing coalesced junk batches
		//
		for ( ; mOutstandingJunkBatches > 0 ; --mOutstandingJunkBatches ) {

		// tell the plugin that all outstanding batches from us
		// have finished.
		//
		rv = junkPlugin->EndBatch();
		NS_ENSURE_SUCCESS(rv, rv);
		}
		}

		return NS_OK;
		}

		// reversing threads involves reversing the threads but leaving the
		// expanded messages ordered relative to the thread, so we
		@@ -5114,7 +5275,6 @@ PRBool nsMsgDBView::OfflineMsgSelected(nsMsgViewIndex * indices, PRInt32 numIndi
		if (localFolder)
		return PR_TRUE;

		nsresult rv = NS_OK;
		for (nsMsgViewIndex index = 0; index < (nsMsgViewIndex) numIndices; index++)
		{
		PRUint32 flags = m_flags.GetAt(indices[index]);