Ignore:
Timestamp:
Sep 14, 2009, 2:12:27 AM (12 years ago)
Author:
Matthäus Wander
Message:

Added new concept of central data files:

  • added new directory "Data" to CT2. It may consist of several subdirectories with different types of data. Currently there is only "wordlists".
  • Plugins can use new class DataManager to automatically loads a list of all available data files of a given type.
  • for each data file there may be an optional file with metainformation. This metafile must have the suffix ".metainfo" to a given existing file name (including file extension).
  • the metafiles are simple text files with a "key=value" format
  • all keys are read by DataManager and made available via class DataMetaFileInfo
  • the user may add additional wordlist files without any code or configuration changes
  • DataManager was ought to be part of CrypCore but it uses the Tokenizer which is part of CrypPluginBase. Therefore the DataManager has been moved to CrypPluginBase for now.

Dictionary:

  • changed Dictionary to use the above DataManager concept
  • added new string[] output
  • marked existing string output as deprecated

WordPatterns:

  • changed dictionary input to use the string[] property
  • adapted project sample to use Dictionary instead of FileInput

AnotherEditor:

  • added QuickWatchView of any array type as text

The deployment of build-in datafiles is currently designed as the following example of Dictionary explains: (subject to change)

  • Create directory Data/wordlists and move data files into there
  • Create metafiles for data files (optional step)
  • Assure data files (and metafiles) have the following properties:
    • Build Action: none
    • Copy: Copy always
  • Change the PostBuild events to the following code snippet: (note that the assembly name has been replaced by a variable, which is useful for all plugins, even if not using data files)
cd "$(ProjectDir)"
cd ..\..\CrypWin\$(OutDir)
if not exist "./CrypPlugins" mkdir "./CrypPlugins"

del /F /S /Q /s /q "$(TargetName)*.*"
copy "$(TargetDir)$(TargetName)*.*" "./CrypPlugins"

if not exist "./Data" mkdir "./Data"
xcopy /Y /S "$(TargetDir)Data" "./Data"

Disclaimer: It's late. This revision may contain errors :-)

Location:
trunk/CrypPlugins/Dictionary
Files:
6 added
2 deleted
3 edited

Legend:

Unmodified
Added
Removed
  • trunk/CrypPlugins/Dictionary/Dictionary.cs

    r419 r511  
    219219namespace Dictionary
    220220{
    221   [Author("Thomas Schmid", "thomas.schmid@cryptool.org", "Uni Siegen", "http://www.uni-siegen.de")]
    222   [PluginInfo(true, "Dictionary", "Reads values from a Dictionary and returns all entries concatenated with given delimiter in one string.", null, "Dictionary/icon.png")]
    223   public class CryptoolDictionary : IInput
    224   {
    225     # region private_variables
    226     private const string myCacheGUID = "96202C52-1EFD-40d3-ABF9-75964BA1E95D";
    227     private const string PluginDirecory = "CrypPlugins";
    228     private CryptoolDictionarySettings settings = new CryptoolDictionarySettings();
    229     private Dictionary<string, string> dicValues;
    230     private readonly string path;
    231     private const string DIC_MARKER = "081C0520-0062-49f2-9832-5F043682F45E";
    232     private List<FileInfo> dicFiles = new List<FileInfo>();
    233     # endregion private_variables
    234 
    235     public List<FileInfo> DicFiles
     221    [Author("Thomas Schmid", "thomas.schmid@cryptool.org", "Uni Siegen", "http://www.uni-siegen.de")]
     222    [PluginInfo(true, "Dictionary", "Reads values from a Dictionary and returns all entries concatenated with given delimiter in one string.", null, "Dictionary/icon.png")]
     223    public class CryptoolDictionary : IInput
    236224    {
    237       get { return dicFiles; }
    238       set { dicFiles = value; }
     225        # region private_variables
     226
     227        private const string DATATYPE = "wordlists";
     228
     229        private CryptoolDictionarySettings settings = new CryptoolDictionarySettings();
     230
     231        // dictionary name -> collection of words
     232        private static Dictionary<DataFileMetaInfo, string[]> dicValues = new Dictionary<DataFileMetaInfo, string[]>();
     233        private static Dictionary<DataFileMetaInfo, string> dicValuesOld = new Dictionary<DataFileMetaInfo, string>();
     234        private static DataFileMetaInfo[] dicList;
     235
     236        private DataManager dataMgr = new DataManager();
     237
     238        # endregion private_variables
     239
     240        public DataFileMetaInfo CurrentDicSelection
     241        {
     242            get
     243            {
     244                if (dicList.Length > settings.Dictionary)
     245                    return dicList[settings.Dictionary];
     246                else
     247                    return null;
     248            }
     249        }
     250
     251        public CryptoolDictionary()
     252        {
     253        }
     254
     255        [Obsolete("Use string[] output instead")]
     256        [PropertyInfo(Direction.OutputData, "Dictionary output (deprecated)", "Values from dictionary as single string (deprecated)", "", false, false, DisplayLevel.Beginner, QuickWatchFormat.Text, null)]
     257        public string OutputString
     258        {
     259            get
     260            {
     261                if (OutputList != null)
     262                {
     263                    Debug.Assert(CurrentDicSelection != null);
     264                    if (!dicValuesOld.ContainsKey(CurrentDicSelection))
     265                        dicValuesOld.Add(CurrentDicSelection, string.Join(" ", OutputList));
     266
     267                    return dicValuesOld[CurrentDicSelection];
     268                }
     269
     270                return null;
     271            }
     272            set { } // readonly
     273        }
     274
     275        [PropertyInfo(Direction.OutputData, "Dictionary Output", "Values from dictionary as array of strings", "", false, false, DisplayLevel.Beginner, QuickWatchFormat.Text, null)]
     276        public string[] OutputList
     277        {
     278            get
     279            {
     280                if (CurrentDicSelection != null)
     281                {
     282                    if (dicValues.ContainsKey(CurrentDicSelection) || LoadDictionary(CurrentDicSelection))
     283                        return dicValues[CurrentDicSelection];
     284                }
     285
     286                return null;
     287            }
     288            set { } // readonly
     289        }
     290
     291        #region IPlugin Members
     292
     293#pragma warning disable 67
     294        public event StatusChangedEventHandler OnPluginStatusChanged;
     295#pragma warning restore
     296
     297        public event GuiLogNotificationEventHandler OnGuiLogNotificationOccured;
     298
     299        public event PluginProgressChangedEventHandler OnPluginProgressChanged;
     300
     301        private void GuiLogMessage(string message, NotificationLevel logLevel)
     302        {
     303            EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(message, this, logLevel));
     304        }
     305
     306        public ISettings Settings
     307        {
     308            get { return settings; }
     309        }
     310
     311        public UserControl Presentation
     312        {
     313            get { return null; }
     314        }
     315
     316        public UserControl QuickWatchPresentation
     317        {
     318            get { return null; }
     319        }
     320
     321        public void PreExecution()
     322        {
     323        }
     324
     325        public void Execute()
     326        {
     327            EventsHelper.ProgressChanged(OnPluginProgressChanged, this, new PluginProgressEventArgs(50, 100));
     328            EventsHelper.PropertyChanged(PropertyChanged, this, new PropertyChangedEventArgs("OutputList"));
     329            EventsHelper.PropertyChanged(PropertyChanged, this, new PropertyChangedEventArgs("OutputString"));
     330            EventsHelper.ProgressChanged(OnPluginProgressChanged, this, new PluginProgressEventArgs(100, 100));
     331        }
     332
     333        public void PostExecution()
     334        {
     335        }
     336
     337        public void Pause()
     338        {
     339        }
     340
     341        public void Stop()
     342        {
     343        }
     344
     345        /// <summary>
     346        /// Loads dictionary file based on current setting.
     347        /// </summary>
     348        /// <returns>true if file has been loaded correctly</returns>
     349        [MethodImpl(MethodImplOptions.Synchronized)]
     350        private bool LoadDictionary(DataFileMetaInfo file)
     351        {
     352            // sanity check for multi-threading
     353            if (dicValues.ContainsKey(file))
     354                return true;
     355
     356            try
     357            {
     358                if (file.DataFile.Exists)
     359                {
     360                    EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(string.Format(Properties.Resources.loading_dic_now, new object[] { file.Name }), this, NotificationLevel.Info));
     361                    Stopwatch stopWatch = new Stopwatch();
     362                    stopWatch.Start();
     363
     364                    FileStream fs = file.DataFile.OpenRead();
     365                    StreamReader sr = new StreamReader(fs);
     366                    List<string> list = new List<string>();
     367                    while (!sr.EndOfStream)
     368                    {
     369                        list.Add(sr.ReadLine());
     370                    }
     371                    dicValues.Add(file, list.ToArray());
     372
     373                    stopWatch.Stop();
     374                    // This log msg is shown on init after first using this plugin, even if event subscription
     375                    // should not have been done yet. Results from using static LoadContent method.
     376                    EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(string.Format(Properties.Resources.finished_loading_dic, new object[] { stopWatch.Elapsed.Milliseconds }), this, NotificationLevel.Info));
     377
     378                    return true;
     379                }
     380                else
     381                {
     382                    EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(string.Format(Properties.Resources.dic_file_not_found, new object[] { file.ToString() }), this, NotificationLevel.Error));
     383                }
     384            }
     385            catch (Exception exception)
     386            {
     387                EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(exception.Message, this, NotificationLevel.Error));
     388            }
     389
     390            return false;
     391        }
     392
     393        public void Initialize()
     394        {
     395            LoadFileList();
     396        }
     397
     398        [MethodImpl(MethodImplOptions.Synchronized)]
     399        private void LoadFileList()
     400        {
     401            dicList = dataMgr.LoadDirectory(DATATYPE).Values.ToArray();
     402
     403            settings.Collection.Clear();
     404            foreach (DataFileMetaInfo meta in dicList)
     405            {
     406                settings.Collection.Add(meta.Name);
     407            }
     408        }
     409
     410        public void Dispose()
     411        {
     412        }
     413
     414        #endregion
     415
     416        #region INotifyPropertyChanged Members
     417
     418        public event PropertyChangedEventHandler PropertyChanged;
     419
     420        #endregion
    239421    }
    240 
    241     public string CurrentDicFilename
    242     {
    243       get
    244       {
    245         try
    246         {
    247           return DicFiles[settings.Dictionary].Name;
    248         }
    249         catch (Exception)
    250         {
    251           return null;
    252         }
    253       }
    254     }
    255 
    256     public CryptoolDictionary()
    257     {
    258       settings.PropertyChanged += settings_PropertyChanged;
    259       path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, PluginDirecory);
    260     }
    261 
    262     /// <summary>
    263     /// Handles the PropertyChanged event of the settings control. If dictionary is changed file will be
    264     /// loaded immediately.
    265     /// </summary>
    266     /// <param name="sender">The source of the event.</param>
    267     /// <param name="e">The <see cref="System.ComponentModel.PropertyChangedEventArgs"/> instance containing the event data.</param>
    268     void settings_PropertyChanged(object sender, PropertyChangedEventArgs e)
    269     {
    270       if (e.PropertyName == "Dictionary")
    271       {
    272         GetCurrentDictionary();
    273       }
    274     }
    275    
    276     [PropertyInfo(Direction.OutputData, "Dictionary Output", "Values from dictionary as string.", "", false, false, DisplayLevel.Beginner, QuickWatchFormat.Text, null)]
    277     public string OutputString
    278     {
    279       get
    280       {
    281         if (dicValues != null && dicValues.ContainsKey(CurrentDicFilename))
    282           return dicValues[CurrentDicFilename];
    283         else if (dicValues != null)
    284         {
    285           GetCurrentDictionary();
    286           if (dicValues.ContainsKey(CurrentDicFilename))
    287             return dicValues[CurrentDicFilename];
    288         }
    289         return null;
    290       }
    291       set { } // readonly
    292     }
    293 
    294     #region IPlugin Members
    295 
    296 #pragma warning disable 67
    297                 public event StatusChangedEventHandler OnPluginStatusChanged;
    298                 public event GuiLogNotificationEventHandler OnGuiLogNotificationOccured;
    299                 public event PluginProgressChangedEventHandler OnPluginProgressChanged;
    300 #pragma warning restore
    301 
    302     private void GuiLogMessage(string message, NotificationLevel logLevel)
    303     {
    304       EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(message, this, logLevel));
    305     }
    306 
    307     public ISettings Settings
    308     {
    309       get { return settings; }
    310     }
    311 
    312     public UserControl Presentation
    313     {
    314       get { return null; }
    315     }
    316 
    317     public UserControl QuickWatchPresentation
    318     {
    319       get { return null; }
    320     }
    321 
    322     public void PreExecution()
    323     {
    324     }
    325 
    326     public void Execute()
    327     {
    328       EventsHelper.ProgressChanged(OnPluginProgressChanged, this, new PluginProgressEventArgs(50, 100));
    329       EventsHelper.PropertyChanged(PropertyChanged, this, new PropertyChangedEventArgs("OutputString"));
    330       EventsHelper.ProgressChanged(OnPluginProgressChanged, this, new PluginProgressEventArgs(100, 100));
    331     }
    332 
    333     public void PostExecution()
    334     {
    335     }
    336 
    337     public void Pause()
    338     {
    339     }
    340 
    341     public void Stop()
    342     {
    343     }
    344 
    345     /// <summary>
    346     /// Gets the dictionary from using AppDomain Data as cache => dics are only loaded once for all plugins
    347     /// of type Dictionary.
    348     /// </summary>
    349     /// <returns></returns>
    350     [MethodImpl(MethodImplOptions.Synchronized)]
    351     private static Dictionary<string, string> GetDictionary()
    352     {
    353       Dictionary<string, string> dic = AppDomain.CurrentDomain.GetData(myCacheGUID) as Dictionary<string, string>;
    354       if (dic == null)
    355       {
    356         dic = new Dictionary<string, string>();
    357         AppDomain.CurrentDomain.SetData(myCacheGUID, dic);
    358       }
    359       return dic;
    360     }
    361 
    362     [MethodImpl(MethodImplOptions.Synchronized)]
    363     private static void LoadContent(string dicKey, string fileName, Dictionary<string, string> dic, char delimiter)
    364     {     
    365       if (!dic.ContainsKey(dicKey))
    366       {       
    367         // string[] theWords = System.IO.File.ReadAllLines(fileName);
    368         StreamReader sr = new StreamReader(File.OpenRead(fileName));
    369         List<string> list = new List<string>();
    370         sr.ReadLine();
    371         sr.ReadLine();
    372         while (!sr.EndOfStream)
    373         {
    374           list.Add(sr.ReadLine());
    375         }       
    376         dic.Add(dicKey, string.Join(delimiter.ToString(), list.ToArray()));
    377       }
    378     }
    379 
    380     /// <summary>
    381     /// Loads dictionary file based on current setting.
    382     /// </summary>
    383     private void GetCurrentDictionary()
    384     {
    385       try
    386       {
    387         dicValues = GetDictionary();
    388         if (dicValues != null && CurrentDicFilename != null && !dicValues.ContainsKey(CurrentDicFilename))
    389         {
    390           string file = Path.Combine(path, CurrentDicFilename);
    391           if (File.Exists(file))
    392           {
    393             EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(string.Format(Properties.Resources.loading_dic_now, new object[] { settings.CurrentDicName }), this, NotificationLevel.Info));
    394             Stopwatch stopWatch = new Stopwatch();
    395             stopWatch.Start();
    396             LoadContent(CurrentDicFilename, file, dicValues, settings.Delimiter[0]);
    397             stopWatch.Stop();
    398             // This log msg is shown on init after first using this plugin, even if event subscription
    399             // should not have been done yet. Results from using static LoadContent method.
    400             EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(string.Format(Properties.Resources.finished_loading_dic, new object[] { stopWatch.Elapsed.Milliseconds }), this, NotificationLevel.Info));           
    401           }
    402           else
    403             EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(string.Format(Properties.Resources.dic_file_not_found, new object[] { file.ToString() }), this, NotificationLevel.Error));
    404         }
    405       }
    406       catch (Exception exception)
    407       {
    408         EventsHelper.GuiLogMessage(OnGuiLogNotificationOccured, this, new GuiLogEventArgs(exception.Message, this, NotificationLevel.Error));
    409       }
    410     }
    411 
    412     public void Initialize()
    413     {
    414       try
    415       {
    416         settings.Collection = new ObservableCollection<string>();
    417         // load dics
    418         DirectoryInfo directory = new DirectoryInfo(path);
    419         foreach (FileInfo fileInfo in directory.GetFiles("*.txt"))
    420         {
    421           try
    422           {
    423             StreamReader sr = new StreamReader(File.OpenRead(fileInfo.FullName));
    424             if (sr.ReadLine().Contains(DIC_MARKER))
    425             {
    426               // seems to be a dic that we want do use...
    427               // if dic description is provided we will use it...
    428               string[] dicName = sr.ReadLine().Split(new string[] { ":" }, StringSplitOptions.RemoveEmptyEntries);
    429               if (dicName.Length == 2)
    430               {
    431                 settings.Collection.Add(dicName[1]);
    432                 DicFiles.Add(fileInfo);
    433               }
    434             }
    435             sr.Close();
    436           }
    437           catch (Exception ex)
    438           {
    439             GuiLogMessage(ex.Message, NotificationLevel.Warning);
    440           }
    441         }
    442 
    443         GetCurrentDictionary();
    444       }
    445       catch (Exception exception)
    446       {
    447         GuiLogMessage(exception.Message, NotificationLevel.Error);
    448       }
    449     }
    450 
    451     public void Dispose()
    452     {
    453     }
    454 
    455     #endregion
    456 
    457     #region INotifyPropertyChanged Members
    458 
    459     public event PropertyChangedEventHandler PropertyChanged;
    460 
    461     #endregion
    462   }
    463422}
  • trunk/CrypPlugins/Dictionary/Dictionary.csproj

    r4 r511  
    8686  </ItemGroup>
    8787  <ItemGroup>
    88     <None Include="Dictionary_wordsGER.txt">
     88    <None Include="Data\wordlists\Dictionary_wordsGER.txt">
    8989      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
    9090    </None>
     
    9595  </ItemGroup>
    9696  <ItemGroup>
    97     <None Include="Dictionary_2of12.txt">
     97    <None Include="Data\wordlists\Dictionary_2of12.txt">
     98      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     99    </None>
     100    <None Include="Data\wordlists\Dictionary_2of12.txt.metainfo">
     101      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     102    </None>
     103    <None Include="Data\wordlists\Dictionary_wordsGER.txt.metainfo">
    98104      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
    99105    </None>
     
    112118if not exist "./CrypPlugins" mkdir "./CrypPlugins"
    113119
    114 del /F /S /Q /s /q "Dictionary*.*"
    115 copy "$(TargetDir)Dictionary*.*" "./CrypPlugins"</PostBuildEvent>
     120del /F /S /Q /s /q "$(TargetName)*.*"
     121copy "$(TargetDir)$(TargetName)*.*" "./CrypPlugins"
     122
     123if not exist "./Data" mkdir "./Data"
     124xcopy /Y /S "$(TargetDir)Data" "./Data"
     125</PostBuildEvent>
    116126  </PropertyGroup>
    117127</Project>
  • trunk/CrypPlugins/Dictionary/DictionarySettings.cs

    r236 r511  
    211211using System.Collections.ObjectModel;
    212212using System.IO;
     213using Cryptool.PluginBase.Miscellaneous;
    213214
    214215namespace Dictionary
     
    221222    # endregion private_variables
    222223
    223     public string CurrentDicName
    224     {
    225       get
    226       {
    227         try
    228         {
    229           return Collection[currentDictionary];
    230         }
    231         catch (Exception)
    232         {
    233           return null;
    234         }
    235       }
    236     }
    237 
    238224    [TaskPane("Dictionary", "Select dictionary with keywords.", "", 0, false, DisplayLevel.Beginner, ControlType.DynamicComboBox, new string[] { "Collection" })]
    239225    public int Dictionary
     
    251237    }
    252238
     239    // CrypWin requires this to be a collection of strings
    253240    public ObservableCollection<string> Collection
    254241    {
     
    264251    }
    265252
    266     private string delimiter = " ";
    267     [TaskPaneAttribute("Delimiter", "The delimiter for the dictionary words (default is blank).", null, 1, false, DisplayLevel.Experienced, ControlType.TextBox, ValidationType.RegEx, "^(.){1,1}$")]
    268     public string Delimiter
    269     {
    270       get { return this.delimiter; }
    271       set
    272       {
    273         if (value != delimiter)
    274         {
    275           delimiter = value;
    276           HasChanges = true;
    277         }
    278       }
    279     }
    280 
    281253    #region ISettings Members
    282254
Note: See TracChangeset for help on using the changeset viewer.