r3dux.org

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
 
namespace SimpleTermFrequencyAnalyser
{
	class Program
	{
		static void Main(string[] args)
		{
			// Read a file into a string (this file must live in the same directory as the executable)
			string filename = "Alice-in-Wonderland.txt";
			string inputString = File.ReadAllText(filename);
 
			// Convert our input to lowercase
			inputString = inputString.ToLower();        
 
			// Define characters to strip from the input and do it
			string[] stripChars = { ";", ",", ".", "-", "_", "^", "(", ")", "[", "]",
						"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\n", "\t", "\r" };
			foreach (string character in stripChars)
			{
				inputString = inputString.Replace(character, "");
			}
 
			// Split on spaces into a List of strings
			List<string> wordList = inputString.Split(' ').ToList();
 
			// Define and remove stopwords
			string[] stopwords = new string[] { "and", "the", "she", "for", "this", "you", "but" };
			foreach (string word in stopwords)
			{
				// While there's still an instance of a stopword in the wordList, remove it.
				// If we don't use a while loop on this each call to Remove simply removes a single
				// instance of the stopword from our wordList, and we can't call Replace on the
				// entire string (as opposed to the individual words in the string) as it's
				// too indiscriminate (i.e. removing 'and' will turn words like 'bandage' into 'bdage'!)
				while ( wordList.Contains(word) )
				{
					wordList.Remove(word);
				}
			}
 
			// Create a new Dictionary object
			Dictionary<string, int> dictionary = new Dictionary<string, int>();
 
			// Loop over all over the words in our wordList...
			foreach (string word in wordList)
			{
				// If the length of the word is at least three letters...
				if (word.Length >= 3) 
				{
					// ...check if the dictionary already has the word.
					if ( dictionary.ContainsKey(word) )
					{
						// If we already have the word in the dictionary, increment the count of how many times it appears
						dictionary[word]++;
					}
					else
					{
						// Otherwise, if it's a new word then add it to the dictionary with an initial count of 1
						dictionary[word] = 1;
					}
 
				} // End of word length check
 
			} // End of loop over each word in our input
 
			// Create a dictionary sorted by value (i.e. how many times a word occurs)
			var sortedDict = (from entry in dictionary orderby entry.Value descending select entry).ToDictionary(pair => pair.Key, pair => pair.Value);
 
			// Loop through the sorted dictionary and output the top 10 most frequently occurring words
			int count = 1;
			Console.WriteLine("---- Most Frequent Terms in the File: " + filename + " ----");
			Console.WriteLine();
			foreach (KeyValuePair<string, int> pair in sortedDict)
			{
				// Output the most frequently occurring words and the associated word counts
				Console.WriteLine(count + "\t" + pair.Key + "\t" + pair.Value);
				count++;
 
				// Only display the top 10 words then break out of the loop!
				if (count > 10)
				{
					break;
				}
			}
 
			// Wait for the user to press a key before exiting
			Console.ReadKey();
 
		} // End of Main method
 
	} // End of Program class
 
} // End of namespace

---- Most Frequent Terms in the File: Alice-in-Wonderland.txt ----
1       said    427
2       was     309
3       alice   262
4       that    211
5       her     204
6       with    185
7       all     161
8       had     158
9       not     130
10      very    123

TF = 20

IDF = log (50 / 15) = log (3.33333) = 0.52288

IDF = log (50 / 1 + 15) = log ( 50 / 16 ) = log ( 3.125 ) = 0.49485

TF*IDF = 20 * 0.49485 = 9.897